def execute(self, sql, args=None):
logger.info(self.mogrify(sql, args))
try:
psycopg2.extensions.cursor.execute(self, sql, args)
except Exception, exc:
logger.error("%s: %s" % (exc.__class__.__name__, exc))
raise
def __exit__(self, exception_type, exception_value, traceback):
if exception_value:
logger.error("Errors were encountered, rolling back session")
self._session.rollback()
else:
self._session.commit()
self._session.close()
logger.info("Closed db session")
def one(self, entity_type, filterr):
query = self._session.query(entity_type).filter_by(**filterr)
try:
entity = query.one()
except NoResultFound:
logger.warn("No results found for %s %s" % (entity_type, filterr))
entity = None
except MultipleResultsFound:
logger.error("MultipleResultsFound")
entity = query.first()
except Exception:
logger.error(sys.exc_info())
entity = None
return entity
def insert_raster(sqa,
config,
datagranule,
ras,
srid_in,
srid_out,
block_size,
mask=None):
raster_format = sqa.one(DataFormat, filterr={"name": "RASTER"})
vector_format = sqa.one(DataFormat, filterr={"name": "VECTOR"})
if (srid_in != srid_out) and datagranule.dataformat_id == raster_format.id:
raise Exception(
"Cannot reproject raster tiles, ensure srid_in and srid_out are same"
)
if datagranule.dataformat_id == raster_format.id:
tile_count = 0
mask_wkt = None
if not mask is None:
mask_wkt = sqa.AsEWKT(mask.geom, srid_in)
for tile in ras.tile_generator(block_size=block_size):
tile_wkt = tile["raster_wkt"]
extent = tile["extent_wkt"]
rt = RasterTile(datagranule=datagranule, rast=tile_wkt)
rt.extent = extent
if mask_wkt:
if sqa.Intersects(extent, mask_wkt):
sqa.insert(rt)
tile_count += 1
else:
sqa.insert(rt)
tile_count += 1
if tile_count and tile_count % 100 == 0:
logger.info("Inserted %d tiles" % tile_count)
logger.info("Inserted %d tiles" % tile_count)
return tile_count
elif datagranule.dataformat_id == vector_format.id:
batch_size = 1000
ogrds = ogr.Open(config.ogr_connection_string())
srs_in = get_srs(srid_in)
if srid_in != srid_out:
srs_out = get_srs(srid_out)
transform = osr.CoordinateTransformation(srs_in, srs_out)
layer = ogrds.CreateLayer(str(datagranule.table_name), srs_out,
ogr.wkbPolygon, ['OVERWRITE=YES'])
else:
transform = None
layer = ogrds.CreateLayer(str(datagranule.table_name), srs_in,
ogr.wkbPolygon, ['OVERWRITE=YES'])
logger.info("Created layer %s" % str(datagranule.table_name))
value_field = ogr.FieldDefn(str(datagranule.variable.name),
ogr.OFTReal)
layer.CreateField(value_field)
layerDefn = layer.GetLayerDefn()
layer.StartTransaction()
count = 0
#create mask to filter ingest geometries
mask_geom = None
if not mask is None:
mask_wkt = sqa.AsWKT(mask.geom, srid_in)
mask_geom = ogr.CreateGeometryFromWkt(mask_wkt)
for items in ras.vector_generator(block_size=block_size):
for item in items:
poly, value = item[0], item[1]
if not mask_geom is None:
if not mask_geom.Intersects(poly):
continue
feature = ogr.Feature(layerDefn)
if transform:
try:
poly.Transform(transform)
feature.SetGeometry(poly)
except:
logger.error("Reprojection failed")
else:
feature.SetGeometry(poly)
feature.SetField(str(datagranule.variable.name), value)
layer.CreateFeature(feature)
count += 1
if count % batch_size == 0:
logger.info("inserted features - %d" % count)
layer.CommitTransaction()
if count % batch_size != 0:
logger.info("inserted features - %d" % count)
layer.CommitTransaction()
return count
else:
raise Exception("Format should be one of raster or vector")
def insert_raster(sqa, config, datagranule, ras, srid_in, srid_out, block_size, mask=None):
raster_format = sqa.one(DataFormat, filterr={"name": "RASTER"})
vector_format = sqa.one(DataFormat, filterr={"name": "VECTOR"})
if (srid_in != srid_out) and datagranule.dataformat_id == raster_format.id:
raise Exception("Cannot reproject raster tiles, ensure srid_in and srid_out are same")
if datagranule.dataformat_id == raster_format.id:
tile_count = 0
mask_wkt = None
if not mask is None:
mask_wkt = sqa.AsEWKT(mask.geom, srid_in)
for tile in ras.tile_generator(block_size=block_size):
tile_wkt = tile["raster_wkt"]
extent = tile["extent_wkt"]
rt = RasterTile(datagranule=datagranule, rast=tile_wkt)
rt.extent = extent
if mask_wkt:
if sqa.Intersects(extent, mask_wkt):
sqa.insert(rt)
tile_count += 1
else:
sqa.insert(rt)
tile_count += 1
if tile_count and tile_count % 100 == 0:
logger.info("Inserted %d tiles" % tile_count)
logger.info("Inserted %d tiles" % tile_count)
return tile_count
elif datagranule.dataformat_id == vector_format.id:
batch_size = 1000
ogrds = ogr.Open(config.ogr_connection_string())
srs_in = get_srs(srid_in)
if srid_in != srid_out:
srs_out = get_srs(srid_out)
transform = osr.CoordinateTransformation(srs_in, srs_out)
layer = ogrds.CreateLayer(str(datagranule.table_name), srs_out, ogr.wkbPolygon, ['OVERWRITE=YES'])
else:
transform = None
layer = ogrds.CreateLayer(str(datagranule.table_name), srs_in, ogr.wkbPolygon, ['OVERWRITE=YES'])
logger.info("Created layer %s" % str(datagranule.table_name))
value_field = ogr.FieldDefn(str(datagranule.variable.name), ogr.OFTReal)
layer.CreateField(value_field)
layerDefn = layer.GetLayerDefn()
layer.StartTransaction()
count = 0
#create mask to filter ingest geometries
mask_geom = None
if not mask is None:
mask_wkt = sqa.AsWKT(mask.geom, srid_in)
mask_geom = ogr.CreateGeometryFromWkt(mask_wkt)
for items in ras.vector_generator(block_size=block_size):
for item in items:
poly, value = item[0], item[1]
if not mask_geom is None:
if not mask_geom.Intersects(poly):
continue
feature = ogr.Feature(layerDefn)
if transform:
try:
poly.Transform(transform)
feature.SetGeometry(poly)
except:
logger.error("Reprojection failed")
else:
feature.SetGeometry(poly)
feature.SetField(str(datagranule.variable.name), value)
layer.CreateFeature(feature)
count += 1
if count % batch_size == 0:
logger.info("inserted features - %d" % count)
layer.CommitTransaction()
if count % batch_size != 0:
logger.info("inserted features - %d" % count)
layer.CommitTransaction()
return count
else:
raise Exception("Format should be one of raster or vector")
