def to_raster(self, filename, driver='GTiff'):
from osgeo import gdal_array
# convert from numpy type to gdal type.
gdal_type = dict((v, k) for (k, v) \
in gdal_array.codes.items())[self.dtype.type]
# make it so we can loop over 3rd axis.
if len(self.shape) == 2:
a = self[:, :, np.newaxis]
else:
a = self
bbox = self.extent
ri = self.agoodle.ri
d = gdal.GetDriverByName(driver)
# NOTE: switch of shape[0] and [1] !!!!
tif = d.Create(filename, a.shape[1], a.shape[0], a.shape[2], gdal_type)
xsize = (bbox[2] - bbox[0]) / (self.shape[1])
ysize = (bbox[1] - bbox[3]) / (self.shape[0])
# could be a bit off since we had to round to pixel.
assert abs(xsize - ri.xsize) < 1
assert abs(ysize - ri.ysize) < 1, (ysize, ri.ysize, xsize, ri.xsize)
tif.SetGeoTransform([bbox[0], xsize, 0, bbox[3], 0, ysize])
for i in range(a.shape[2]):
b = tif.GetRasterBand(i + 1)
ct = self.agoodle.raster.GetRasterBand(i + 1).GetRasterColorTable()
if ct:
b.SetRasterColorTable(ct)
gdal_array.BandWriteArray(b, a[:, :, i])
tif.SetProjection(self.agoodle.raster.GetProjection())
return tif
def create_tif_mask(in_fname, out_fname):
# Open the raster file
raster = gdal.Open(in_fname, gdal.GA_ReadOnly)
band_1 = raster.GetRasterBand(1)
# Read the data as a numpy array
data_1 = gdal_array.BandReadAsArray(band_1)
# Create a boolean band with all 1's
mask = (data_1 >= 0).astype(int)
assert np.mean(mask) == 1
# Write the output mask
driver = gdal.GetDriverByName("GTiff")
ds_out = driver.Create(out_fname, raster.RasterXSize, raster.RasterYSize,
1, band_1.DataType)
gdal_array.CopyDatasetInfo(raster, ds_out)
band_out = ds_out.GetRasterBand(1)
gdal_array.BandWriteArray(band_out, mask)
# Close the datasets
band_1 = None
raster = None
band_out = None
ds_out = None
def pixelProfileToRaster(self,
idx_row=0,
idx_col=0,
currentPixelArray=None):
''' Saves the current temporal profile for selected pixel @(idx_row,idx_col)
to a previous created empty raster '''
if self.out_filename == None:
self.update(
) # It will updated itself if no out filename is given.
self.status = None
print('updating pixel in', self.out_filename + self.outfileExt)
fullpath = self.out_path + self.out_filename + self.outfileExt
updatedataset = og.Open(fullpath, ogc.GA_Update)
if currentPixelArray == None:
currentPixelArray = self.getPixelProfile(idx_row, idx_col)
nbands = currentPixelArray.shape[0]
for iband in range(nbands):
eprofile = np.array(currentPixelArray[iband])
eprofile.shape = (1, -1) #shape (1,1)
currentBand = updatedataset.GetRasterBand(iband + 1)
oga.BandWriteArray(currentBand,
eprofile,
xoff=idx_col,
yoff=idx_row)
pass
updatedataset = None
self.status = True
pass
def write_array_to_file(dst_filename: PathLikeOrStr,
a: MaybeSequence[np.ndarray],
gdal_dtype=None) -> gdal.Dataset:
driver_name = GetOutputDriverFor(dst_filename, is_raster=True)
driver = gdal.GetDriverByName(driver_name)
a_shape = a[0].shape
if len(a_shape) == 1:
# 2d array, singleband raster
a = [a]
bands_count = 1
elif len(a_shape) == 2:
# 3d array, multiband raster
bands_count = a.shape[0]
else:
raise Exception('Array should have 2 or 3 dimensions')
y_size, x_size = a[0].shape
if gdal_dtype is None:
np_dtype = a[0].dtype
gdal_dtype = gdal_array.flip_code(np_dtype)
ds = driver.Create(dst_filename, x_size, y_size, bands_count, gdal_dtype)
if ds is None:
raise Exception(f'failed to create: {dst_filename}')
for bnd_num in range(bands_count):
bnd = ds.GetRasterBand(bnd_num + 1)
if gdal_array.BandWriteArray(bnd, a[bnd_num], xoff=0, yoff=0) != 0:
raise Exception('I/O error')
return ds
def setUp(self):
self.band = numpy.array([[0, 1], [2, 3]], dtype=numpy.uint16)
self.mask = numpy.array([[0, 1], [0, 1]], dtype=numpy.bool)
self.metadata = {'geotransform': (-1.0, 2.0, 0.0, 1.0, 0.0, -1.0)}
self.test_photometric_alpha_image = 'test_photometric_alpha_image.tif'
test_ds = gdal.GetDriverByName('GTiff').Create(
self.test_photometric_alpha_image,
2,
2,
4,
gdal.GDT_UInt16,
options=['PHOTOMETRIC=RGB', 'ALPHA=YES'])
gdal_array.BandWriteArray(test_ds.GetRasterBand(1), self.band)
gdal_array.BandWriteArray(test_ds.GetRasterBand(2), self.band)
gdal_array.BandWriteArray(test_ds.GetRasterBand(3), self.band)
gdal_array.BandWriteArray(test_ds.GetRasterBand(4), self.mask)
test_ds.SetGeoTransform(self.metadata['geotransform'])
def polygonize(shapepath, file, rasterpath):
srcarray,projection,geotrans, shape = GetRasterDataSource(file, rasterpath)
print "Start polygonizing.."
start = time.time()
srcarray[srcarray > 1] = 0 #we are interested in the class 1, we put everything else to 0
drv = gdal.GetDriverByName('MEM')
srs = osr.SpatialReference()
srs.ImportFromWkt(projection)
src_ds = drv.Create('', shape[1], shape[0], 1, gdal.GDT_UInt16)
src_ds.SetGeoTransform(geotrans)
src_ds.SetProjection(projection)
# gdal_array.BandWriteArray(src_ds.GetRasterBand(1), srcarray.T)
gdal_array.BandWriteArray(src_ds.GetRasterBand(1), srcarray)
srcband = src_ds.GetRasterBand(1)
drv = ogr.GetDriverByName("ESRI Shapefile")
dst_ds = drv.CreateDataSource(shapepath + file + ".shp")
dst_layer = dst_ds.CreateLayer(shapepath + file, srs = srs)
new_field = ogr.FieldDefn("type", ogr.OFTInteger)
dst_layer.CreateField(new_field)
new_field = None
new_field = ogr.FieldDefn("area", ogr.OFTReal)
# new_field.SetWidth(32)
# new_field.SetPrecision(2) #added line to set precision (for floating point)
dst_layer.CreateField(new_field)
gdal.Polygonize(srcband, srcband, dst_layer, 0, [], callback = None )
print "Number of features detected: ", dst_layer.GetFeatureCount()
for feature in dst_layer:
geom = feature.GetGeometryRef()
area = geom.GetArea()
feature.SetField("area", area)
dst_layer.SetFeature(feature)
# Filter per size
if (geom.GetArea() < 1) or (geom.GetArea() > 100):
dst_layer.DeleteFeature(feature.GetFID())
dst_layer.SyncToDisk()
dst_ds.ExecuteSQL("REPACK " + file)
print "Number of features after deleting: ", dst_layer.GetFeatureCount()
dst_ds = None
print "*---------------------------*"
print "Shapefile correctly saved in: " + shapepath
end = time.time()
print "Time for polygonizing: "
print (end-start)
def writeFloatRaster(outpath,inArray,template_DS,FLOAT_NODATA=-9999,mode=0777):#writes a single band raster to the same extent and projection as the template DS image_driver=template_DS.GetDriver() image_out_DS=image_driver.Create(outpath,template_DS.RasterXSize,template_DS.RasterYSize,template_DS.RasterCount,GDT_Float32) projection=template_DS.GetProjectionRef() geotransform=template_DS.GetGeoTransform() image_out_DS.SetProjection(projection) image_out_DS.SetGeoTransform(geotransform) image_band_out=image_out_DS.GetRasterBand(1) image_band_out.SetNoDataValue(FLOAT_NODATA) image_band_out=gdal_array.BandWriteArray(image_band_out,inArray) os.chmod(outpath,mode) image_out_DS=None
def RasterDifference(RasterFile1,
RasterFile2,
raster_band=1,
OutFileName="Test.outfile",
OutFileType="ENVI"):
"""
Takes two rasters of same size and subtracts second from first,
e.g. Raster1 - Raster2 = raster_of_difference
then writes it out to file
"""
Raster1 = gdal.Open(RasterFile1)
Raster2 = gdal.Open(RasterFile2)
print("RASTER 1: ")
print(Raster1.GetGeoTransform())
print(Raster1.RasterCount)
print(Raster1.GetRasterBand(1).XSize)
print(Raster1.GetRasterBand(1).YSize)
print(Raster1.GetRasterBand(1).DataType)
print("RASTER 2: ")
print(Raster2.GetGeoTransform())
print(Raster2.RasterCount)
print(Raster2.GetRasterBand(1).XSize)
print(Raster2.GetRasterBand(1).YSize)
print(Raster2.GetRasterBand(1).DataType)
raster_array1 = np.array(Raster1.GetRasterBand(raster_band).ReadAsArray())
raster_array2 = np.array(Raster2.GetRasterBand(raster_band).ReadAsArray())
assert (raster_array1.shape == raster_array2.shape)
print("Shapes: ", raster_array1.shape, raster_array2.shape)
difference_raster_array = raster_array1 - raster_array2
# import matplotlib.pyplot as plt
#
# plt.imshow(difference_raster_array)
#
driver = gdal.GetDriverByName(OutFileType)
dsOut = driver.Create(OutFileName,
Raster1.GetRasterBand(1).XSize,
Raster1.GetRasterBand(1).YSize, 1, gdal.GDT_Float32)
#Raster1.GetRasterBand(raster_band).DataType)
gdal_array.CopyDatasetInfo(Raster1, dsOut)
bandOut = dsOut.GetRasterBand(1)
gdal_array.BandWriteArray(bandOut, difference_raster_array)
def save(self, filename):
band_count = len(self.bands)
ysize, xsize = self.bands[0].shape
options = []
if band_count == 3:
options.append('PHOTOMETRIC=RGB')
if self.alpha is not None:
band_count += 1
options.append('ALPHA=YES')
datatype = gdal_array.NumericTypeCodeToGDALTypeCode(
self.bands[0].dtype.type)
gdal_ds = gdal.GetDriverByName('GTIFF').Create(filename,
xsize,
ysize,
band_count,
datatype,
options=options)
# Save georeferencing information
if 'projection' in self.metadata.keys():
gdal_ds.SetProjection(self.metadata['projection'])
if 'geotransform' in self.metadata.keys():
gdal_ds.SetGeoTransform(self.metadata['geotransform'])
if 'rpc' in self.metadata.keys():
gdal_ds.SetMetadata(self.metadata['rpc'], 'RPC')
for i in range(len(self.bands)):
gdal_array.BandWriteArray(gdal_ds.GetRasterBand(i + 1),
self.bands[i])
if self.alpha is not None:
alpha = self.alpha
alpha_band = gdal_ds.GetRasterBand(gdal_ds.RasterCount)
# To conform to 16 bit TIFF alpha expectations transform
# alpha to 16bit.
if alpha_band.DataType == gdal.GDT_UInt16:
alpha = ((alpha.astype(np.uint32) * 65535) / 255).astype(
np.uint16)
def rowProfileToImage(
self,
idx_row=0,
currentRowArray=None,
rasterFormat='ERDAS'): #currentRowArray.shape (nbands,1 row,ncols)
''' Saves the current temporal profile for selected pixel @(idx_row,idx_col)
to a previous created empty raster '''
def update():
self.out_filename = self.name[:-4] + '_new'
self.outfileExt = self.name[-4:]
if self.out_path == None:
self.out_path = 'D://My Docs//working//out_test//'
self.create_emptyRaster(out_path=self.out_path,\
filenameWithoutExtension=self.out_filename,\
outputRasterFormat=rasterFormat)
pass
if self.out_filename == None:
update() # It will updated itself if no out filename is given.
if (self.name == (self.out_filename + self.outfileExt)):
print('Creating an empty raster')
update()
self.status = None
print('updating row in', self.out_filename + self.outfileExt)
fullpath = self.out_path + self.out_filename + self.outfileExt
updatedataset = og.Open(fullpath, ogc.GA_Update)
if currentRowArray == None: # If array in not given, retrieve the one @ given row. This allows to write other arrays instead of the image.
currentRowArray = self.getRowProfile(idx_row)
pass
for iband in range(currentRowArray.shape[0]):
currentBand = updatedataset.GetRasterBand(iband + 1)
rowprofile = currentRowArray[iband] # (1 row,ncols)
oga.BandWriteArray(currentBand, rowprofile, xoff=0, yoff=idx_row)
updatedataset = None
self.status = True
pass
def _create_image_with_geotransform(self, filename):
gdal_ds = gdal.GetDriverByName('GTiff').Create(filename, 2, 2, 1,
gdal.GDT_UInt16)
gdal_array.BandWriteArray(gdal_ds.GetRasterBand(1),
np.ones((2, 2), dtype=np.uint16))
gdal_ds.SetGeoTransform([-1.0, 1.0, 0.0, 1.0, 0.0, -1.0])
def process(parsed, target, temp_metatile, temp_processed, save_offsetx,
save_offsety, save_xsize, save_ysize, nodata, ot, *args, **kwargs):
_, gt, _, nodata, array_numpy = numpy_read(temp_metatile)
#target_db = target.split(".")[0] + ".sqlite"
targetdb = "geodata"
dbuser = "******"
targettable = str(parsed.table)
median = int(parsed.median)
# geotransform values for tile
xmin = gt[0] + (save_offsetx * gt[1])
xmax = gt[0] + ((save_offsetx + save_xsize) * gt[1])
ymin = gt[3] + ((save_offsety + save_ysize) * gt[5])
ymax = gt[3] + (save_offsety * gt[5])
# geotransform values for metatile
save_offsetx = 0
save_offsety = 0
save_xsize = array_numpy.shape[1]
save_ysize = array_numpy.shape[0]
# reclassify
'''
11: cropland
14: cropland
20: cropland
30: forest
40: forest
50: forest
60: forest
70: forest
90: forest
100: forest
110: shrubland
120: shrubland
130: shrubland
140: shrubland
150: shrubland
160: forest
170: forest
180: grassland
190: sealed
200: bare
210: water
220: ice
230: nodata
'''
'''
nodata: 0
cropland: 1
forest: 2
shrubland: 3
grassland: 4
sealed: 5
bare: 6
water: 7
ice: 8
'''
# vectorize (smooth edges)
# smooth
# reclassify
############
classification = {
11: 1,
14: 1,
20: 1,
30: 2,
40: 2,
50: 2,
60: 2,
70: 2,
90: 2,
100: 2,
110: 3,
120: 3,
130: 3,
140: 3,
150: 3,
160: 2,
170: 2,
180: 4,
190: 5,
200: 6,
210: 7,
220: 8 #,
#230:0
}
temp_numpy = numpy.zeros((array_numpy.shape[0], array_numpy.shape[1]))
for c in classification:
idx = (array_numpy == c).nonzero()
temp_numpy[idx] = classification[c]
# median filter
###############
processed_numpy = ndimage.median_filter(temp_numpy, size=median)
#create Memory driver
src_ds = gdal.Open(temp_metatile)
format = "MEM"
driver = gdal.GetDriverByName(format)
mem_ds = driver.CreateCopy('', src_ds)
# write filtered numpy array to GDAL band
gdal_array.BandWriteArray(mem_ds.GetRasterBand(1), processed_numpy)
mem_ds.GetRasterBand(1).WriteArray(processed_numpy)
drv = ogr.GetDriverByName('Memory')
ogr_ds = drv.CreateDataSource('out')
ogr_lyr = ogr_ds.CreateLayer('landcover')
fieldname = 'type'
field_defn = ogr.FieldDefn(fieldname, ogr.OFTInteger)
ogr_lyr.CreateField(field_defn)
ogr_field = ogr_lyr.GetLayerDefn().GetFieldIndex(fieldname)
ogr_field = 0
maskband = None
gdal.Polygonize(mem_ds.GetRasterBand(1), maskband, ogr_lyr, ogr_field, [])
# clip to tile boundary
#######################
ring = ogr.Geometry(ogr.wkbLinearRing)
ring.AddPoint(xmin, ymin)
ring.AddPoint(xmin, ymax)
ring.AddPoint(xmax, ymax)
ring.AddPoint(xmax, ymin)
ring.AddPoint(xmin, ymin)
clipbox = ogr.Geometry(ogr.wkbPolygon)
clipbox.AddGeometry(ring)
cliplayer = ogr_ds.CreateLayer('clipbox', geom_type=ogr.wkbPolygon)
clipfeaturedefn = cliplayer.GetLayerDefn()
clipfeature = ogr.Feature(clipfeaturedefn)
clipfeature.SetGeometry(clipbox)
cliplayer.CreateFeature(clipfeature)
clipped = ogr_ds.CreateLayer('landcover_clipped',
geom_type=ogr.wkbMultiPolygon)
#clipped.ForceToMultiLineString()
field_defn = ogr.FieldDefn('ID', ogr.OFTInteger)
clipped.CreateField(field_defn)
field_defn = ogr.FieldDefn('type', ogr.OFTInteger)
clipped.CreateField(field_defn)
ogr_lyr.Clip(cliplayer, clipped)
connection = psycopg2.connect(database=targetdb, user=dbuser)
# psql -d geodata -c "DROP TABLE landcover; CREATE TABLE landcover (id bigserial primary key, typeid double precision, type text); SELECT AddGeometryColumn ('','landcover','the_geom',4326,'MULTIPOLYGON',2);"
cursor = connection.cursor()
for i in range(clipped.GetFeatureCount()):
insert = True
feature = clipped.GetFeature(i)
geometry = feature.GetGeometryRef()
#print geometry.GetGeometryName()
if geometry.GetGeometryName() in ("POLYGON", "MULTIPOLYGON"):
continue
#print "polygon"
if geometry.GetGeometryName() == "GEOMETRYCOLLECTION":
geometry_new = ogr.Geometry(ogr.wkbMultiLineString)
for i in xrange(geometry.GetGeometryCount()):
g = geometry.GetGeometryRef(i)
if g.GetGeometryName() in ("POLYGON", "MULTIPOLYGON"):
#print "geometrycollection polygon"
geometry_new.AddGeometry(g.Clone())
else:
print g.GetGeometryName()
geometry = geometry_new
if geometry.GetGeometryName() in ("LINESTRING", "MULTILINESTRING",
"POINT", "MULTIPOINT"):
insert = False
if insert:
lctype = feature.GetField("type")
geometry.SetCoordinateDimension(2)
wkt = geometry.ExportToWkt()
cursor.execute(
"INSERT INTO " + targettable +
" (type,the_geom) VALUES (%s, ST_Multi(ST_GeomFromText(%s, " +
"4326)))", (lctype, wkt))
connection.commit()
ogr_ds.Destroy()
processed_numpy = []
mem_ds = None
#numpy_save(processed_numpy, target, save_offsetx, save_offsety, save_xsize, save_ysize, gt, nodata, ot)
#processed_numpy = array_numpy
#print "save processed_numpy"
#print str(processed_numpy.shape[0]) + " " + str(processed_numpy.shape[1])
cursor.close()
connection.close()
def save_alpha_band(gdal_ds, alpha_array):
alpha_band = gdal_ds.GetRasterBand(gdal_ds.RasterCount)
alpha_band.SetColorInterpretation(gdal.GCI_AlphaBand)
gdal_array.BandWriteArray(alpha_band,
alpha_array.astype(numpy.uint16) * 255)
def save_band(gdal_ds, band_array, band_no, nodata=None):
gdal_band = gdal_ds.GetRasterBand(band_no)
gdal_array.BandWriteArray(gdal_band, band_array)
if nodata is not None:
gdal_band.SetNoDataValue(nodata)
def process(parsed, target, temp_metatile, temp_processed, save_offsetx, save_offsety, save_xsize, save_ysize, nodata, ot, *args, **kwargs):
#target_db = target.split(".")[0] + ".sqlite"
targetdb = "geodata"
dbuser = "******"
targettable = str(parsed.table)
elevation = int(parsed.elevation)
median = int(parsed.median)
glacier_mask = parsed.glacier_mask
nodata = 0
#print "read temp_metatile"
_, gt, _, nodata, array_numpy = numpy_read(temp_metatile)
processed_numpy = ndimage.median_filter(array_numpy, size=median)
#processed_numpy = array_numpy
#print "save processed_numpy"
#print str(processed_numpy.shape[0]) + " " + str(processed_numpy.shape[1])
# geotransform values for tile
xmin = gt[0] + (save_offsetx * gt[1])
xmax = gt[0] + ((save_offsetx + save_xsize) * gt[1])
ymin = gt[3] + ((save_offsety + save_ysize) * gt[5])
ymax = gt[3] + (save_offsety * gt[5])
# geotransform values for metatile
save_offsetx = 0
save_offsety = 0
save_xsize = processed_numpy.shape[1]
save_ysize = processed_numpy.shape[0]
# create contours from processed_numpy
ds = gdal.Open(temp_metatile)
drv = ogr.GetDriverByName( 'Memory' )
ogr_ds = drv.CreateDataSource('out')
#ogr_ds = ogr.GetDriverByName('ESRI Shapefile').CreateDataSource('contours_ogr/contour.shp')
ogr_lyr = ogr_ds.CreateLayer('contour', geom_type = ogr.wkbMultiLineString)
field_defn = ogr.FieldDefn('ID', ogr.OFTInteger)
ogr_lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('elev', ogr.OFTReal)
ogr_lyr.CreateField(field_defn)
#create Memory driver
src_ds = gdal.Open( temp_metatile )
format = "MEM"
driver = gdal.GetDriverByName( format )
mem_ds = driver.CreateCopy('', src_ds )
# write filtered numpy array to GDAL band
gdal_array.BandWriteArray(mem_ds.GetRasterBand(1), processed_numpy)
mem_ds.GetRasterBand(1).WriteArray(processed_numpy)
# write contours to OGR layer
gdal.ContourGenerate(mem_ds.GetRasterBand(1), elevation, 0, [], 0, 0, ogr_lyr, 0, 1)
# convert 3D geometries to 2D
for i in range(ogr_lyr.GetFeatureCount()):
feature = ogr_lyr.GetFeature(i)
geometry = feature.GetGeometryRef()
geometry.SetCoordinateDimension(2)
# clip to tile boundary
ring = ogr.Geometry(ogr.wkbLinearRing)
ring.AddPoint(xmin, ymin)
ring.AddPoint(xmin, ymax)
ring.AddPoint(xmax, ymax)
ring.AddPoint(xmax, ymin)
ring.AddPoint(xmin, ymin)
clipbox = ogr.Geometry(ogr.wkbPolygon)
clipbox.AddGeometry(ring)
cliplayer = ogr_ds.CreateLayer('clipbox', geom_type=ogr.wkbPolygon)
clipfeaturedefn = cliplayer.GetLayerDefn()
clipfeature = ogr.Feature(clipfeaturedefn)
clipfeature.SetGeometry(clipbox)
cliplayer.CreateFeature(clipfeature)
clipped = ogr_ds.CreateLayer('contour_clipped', geom_type=ogr.wkbMultiLineString)
#clipped.ForceToMultiLineString()
field_defn = ogr.FieldDefn('ID', ogr.OFTInteger)
clipped.CreateField(field_defn)
field_defn = ogr.FieldDefn('elev', ogr.OFTReal)
clipped.CreateField(field_defn)
ogr_lyr.Clip(cliplayer, clipped)
# PostGIS connection
#connection = db.connect('contours.sqlite')
connection = psycopg2.connect(database = targetdb, user = dbuser)
# psql -d geodata -c "DROP TABLE contours; CREATE TABLE contours (id bigserial primary key, elev double precision, type text); SELECT AddGeometryColumn ('','contours','the_geom',4326,'MULTILINESTRING',2);"
cursor = connection.cursor()
if glacier_mask:
# read glacier_mask
shapefile = glacier_mask
driver = ogr.GetDriverByName("ESRI Shapefile")
dataSource = driver.Open(shapefile, 0)
glacier_layer = dataSource.GetLayer()
glacier_layer_clipped = ogr_ds.CreateLayer('glacier_clipped', geom_type=ogr.wkbPolygon)
glacier_layer.Clip(cliplayer, glacier_layer_clipped)
#print "processing land contours"
# clip & save land contours
land_clipped = ogr_ds.CreateLayer('contour_clipped', geom_type=ogr.wkbMultiLineString)
field_defn = ogr.FieldDefn('ID', ogr.OFTInteger)
land_clipped.CreateField(field_defn)
field_defn = ogr.FieldDefn('elev', ogr.OFTReal)
land_clipped.CreateField(field_defn)
if glacier_layer_clipped.GetFeatureCount() == 0:
land_clipped = clipped
else:
# create inverse clip
inverse_clip = ogr_ds.CreateLayer('inverse_clip', geom_type=ogr.wkbPolygon)
cliplayer.Erase(glacier_layer_clipped, inverse_clip)
clipped.Clip(inverse_clip, land_clipped)
contour_type = "land"
for i in range(land_clipped.GetFeatureCount()):
feature = land_clipped.GetFeature(i)
geometry = feature.GetGeometryRef()
# hack removing loose points from geometry
if geometry.GetGeometryName() in ("POINT", "MULTIPOINT") :
continue
if geometry.GetGeometryName() == "GEOMETRYCOLLECTION" :
geometry_new = ogr.Geometry(ogr.wkbMultiLineString)
for i in xrange(geometry.GetGeometryCount()):
g = geometry.GetGeometryRef(i)
if g.GetGeometryName() == "LINESTRING" :
geometry_new.AddGeometry(g.Clone())
geometry = geometry_new
elev = feature.GetField("elev")
geometry.SetCoordinateDimension(2)
wkt = geometry.ExportToWkt()
cursor.execute("INSERT INTO " + targettable + " (elev,the_geom,type) VALUES (%s, ST_Multi(ST_GeomFromText(%s, " +"4326)), %s)", (elev, wkt, contour_type))
connection.commit()
#print "processing glacier contours"
# clip & save glacier contours
glacier_clipped = ogr_ds.CreateLayer('glacier_clipped', geom_type=ogr.wkbMultiLineString)
field_defn = ogr.FieldDefn('ID', ogr.OFTInteger)
glacier_clipped.CreateField(field_defn)
field_defn = ogr.FieldDefn('elev', ogr.OFTReal)
glacier_clipped.CreateField(field_defn)
clipped.Clip(glacier_layer_clipped, glacier_clipped)
contour_type = "glaciated"
for i in range(glacier_clipped.GetFeatureCount()):
feature = glacier_clipped.GetFeature(i)
geometry = feature.GetGeometryRef()
# hack removing loose points from geometry
if geometry.GetGeometryName() in ("POINT", "MULTIPOINT") :
continue
if geometry.GetGeometryName() == "GEOMETRYCOLLECTION" :
geometry_new = ogr.Geometry(ogr.wkbMultiLineString)
for i in xrange(geometry.GetGeometryCount()):
g = geometry.GetGeometryRef(i)
if g.GetGeometryName() == "LINESTRING" :
geometry_new.AddGeometry(g.Clone())
geometry = geometry_new
elev = feature.GetField("elev")
geometry.SetCoordinateDimension(2)
wkt = geometry.ExportToWkt()
cursor.execute("INSERT INTO " + targettable + " (elev,the_geom,type) VALUES (%s, ST_Multi(ST_GeomFromText(%s, " +"4326)), %s)",(elev, wkt, contour_type))
connection.commit()
else:
# save to POSTGIS
for i in range(clipped.GetFeatureCount()):
feature = clipped.GetFeature(i)
geometry = feature.GetGeometryRef()
# hack removing loose points from geometry
if geometry.GetGeometryName() in ("POINT", "MULTIPOINT") :
continue
if geometry.GetGeometryName() == "GEOMETRYCOLLECTION" :
geometry_new = ogr.Geometry(ogr.wkbMultiLineString)
for i in xrange(geometry.GetGeometryCount()):
g = geometry.GetGeometryRef(i)
if g.GetGeometryName() == "LINESTRING" :
geometry_new.AddGeometry(g.Clone())
geometry = geometry_new
elev = feature.GetField("elev")
#feature_geometry = ogr.ForceToMultiLineString(feature.GetGeometryRef())
geometry.SetCoordinateDimension(2)
wkt = geometry.ExportToWkt()
cursor.execute("INSERT INTO " + targettable + " (elev,the_geom) VALUES (%s, ST_Multi(ST_GeomFromText(%s, " +"4326)))", (elev, wkt))
connection.commit()
ogr_ds.Destroy()
processed_numpy = []
mem_ds = None
#os.remove(target)
#os.remove(temp_target)
cursor.close()
connection.close()
def Calc(calc: MaybeSequence[str], outfile: Optional[PathLikeOrStr] = None, NoDataValue: Optional[Number] = None,
type: Optional[Union[GDALDataType, str]] = None, format: Optional[str] = None,
creation_options: Optional[Sequence[str]] = None, allBands: str = '', overwrite: bool = False,
hideNoData: bool = False, projectionCheck: bool = False,
color_table: Optional[ColorTableLike] = None,
extent: Optional[Extent] = None, projwin: Optional[Union[Tuple, GeoRectangle]] = None,
user_namespace: Optional[Dict]=None,
debug: bool = False, quiet: bool = False, **input_files):
if debug:
print(f"gdal_calc.py starting calculation {calc}")
# Single calc value compatibility
if isinstance(calc, (list, tuple)):
calc = calc
else:
calc = [calc]
calc = [c.strip('"') for c in calc]
creation_options = creation_options or []
# set up global namespace for eval with all functions of gdal_array, numpy
global_namespace = {key: getattr(module, key)
for module in [gdal_array, numpy] for key in dir(module) if not key.startswith('__')}
if user_namespace:
global_namespace.update(user_namespace)
if not calc:
raise Exception("No calculation provided.")
elif not outfile and format.upper() != 'MEM':
raise Exception("No output file provided.")
if format is None:
format = GetOutputDriverFor(outfile)
if isinstance(extent, GeoRectangle):
pass
elif projwin:
if isinstance(projwin, GeoRectangle):
extent = projwin
else:
extent = GeoRectangle.from_lurd(*projwin)
elif not extent:
extent = Extent.IGNORE
else:
extent = extent_util.parse_extent(extent)
compatible_gt_eps = 0.000001
gt_diff_support = {
GT.INCOMPATIBLE_OFFSET: extent != Extent.FAIL,
GT.INCOMPATIBLE_PIXEL_SIZE: False,
GT.INCOMPATIBLE_ROTATION: False,
GT.NON_ZERO_ROTATION: False,
}
gt_diff_error = {
GT.INCOMPATIBLE_OFFSET: 'different offset',
GT.INCOMPATIBLE_PIXEL_SIZE: 'different pixel size',
GT.INCOMPATIBLE_ROTATION: 'different rotation',
GT.NON_ZERO_ROTATION: 'non zero rotation',
}
################################################################
# fetch details of input layers
################################################################
# set up some lists to store data for each band
myFileNames = [] # input filenames
myFiles = [] # input DataSets
myBands = [] # input bands
myAlphaList = [] # input alpha letter that represents each input file
myDataType = [] # string representation of the datatype of each input file
myDataTypeNum = [] # datatype of each input file
myNDV = [] # nodatavalue for each input file
DimensionsCheck = None # dimensions of the output
Dimensions = [] # Dimensions of input files
ProjectionCheck = None # projection of the output
GeoTransformCheck = None # GeoTransform of the output
GeoTransforms = [] # GeoTransform of each input file
GeoTransformDiffer = False # True if we have inputs with different GeoTransforms
myTempFileNames = [] # vrt filename from each input file
myAlphaFileLists = [] # list of the Alphas which holds a list of inputs
# loop through input files - checking dimensions
for alphas, filenames in input_files.items():
if isinstance(filenames, (list, tuple)):
# alpha is a list of files
myAlphaFileLists.append(alphas)
elif is_path_like(filenames) or isinstance(filenames, gdal.Dataset):
# alpha is a single filename or a Dataset
filenames = [filenames]
alphas = [alphas]
else:
# I guess this alphas should be in the global_namespace,
# It would have been better to pass it as user_namepsace, but I'll accept it anyway
global_namespace[alphas] = filenames
continue
for alpha, filename in zip(alphas * len(filenames), filenames):
if not alpha.endswith("_band"):
# check if we have asked for a specific band...
alpha_band = f"{alpha}_band"
if alpha_band in input_files:
myBand = input_files[alpha_band]
else:
myBand = 1
myF_is_ds = not is_path_like(filename)
if myF_is_ds:
myFile = filename
filename = None
else:
myFile = open_ds(filename, gdal.GA_ReadOnly)
if not myFile:
raise IOError(f"No such file or directory: '{filename}'")
myFileNames.append(filename)
myFiles.append(myFile)
myBands.append(myBand)
myAlphaList.append(alpha)
dt = myFile.GetRasterBand(myBand).DataType
myDataType.append(gdal.GetDataTypeName(dt))
myDataTypeNum.append(dt)
myNDV.append(None if hideNoData else myFile.GetRasterBand(myBand).GetNoDataValue())
# check that the dimensions of each layer are the same
myFileDimensions = [myFile.RasterXSize, myFile.RasterYSize]
if DimensionsCheck:
if DimensionsCheck != myFileDimensions:
GeoTransformDiffer = True
if extent in [Extent.IGNORE, Extent.FAIL]:
raise Exception(
f"Error! Dimensions of file {filename} ({myFileDimensions[0]:d}, "
f"{myFileDimensions[1]:d}) are different from other files "
f"({DimensionsCheck[0]:d}, {DimensionsCheck[1]:d}). Cannot proceed")
else:
DimensionsCheck = myFileDimensions
# check that the Projection of each layer are the same
myProjection = myFile.GetProjection()
if ProjectionCheck:
if projectionCheck and ProjectionCheck != myProjection:
raise Exception(
f"Error! Projection of file {filename} {myProjection} "
f"are different from other files {ProjectionCheck}. Cannot proceed")
else:
ProjectionCheck = myProjection
# check that the GeoTransforms of each layer are the same
myFileGeoTransform = myFile.GetGeoTransform(can_return_null=True)
if extent == Extent.IGNORE:
GeoTransformCheck = myFileGeoTransform
else:
Dimensions.append(myFileDimensions)
GeoTransforms.append(myFileGeoTransform)
if not GeoTransformCheck:
GeoTransformCheck = myFileGeoTransform
else:
my_gt_diff = extent_util.gt_diff(GeoTransformCheck, myFileGeoTransform, eps=compatible_gt_eps,
diff_support=gt_diff_support)
if my_gt_diff not in [GT.SAME, GT.ALMOST_SAME]:
GeoTransformDiffer = True
if my_gt_diff != GT.COMPATIBLE_DIFF:
raise Exception(
f"Error! GeoTransform of file {filename} {myFileGeoTransform} is incompatible "
f"({gt_diff_error[my_gt_diff]}), first file GeoTransform is {GeoTransformCheck}. "
f"Cannot proceed")
if debug:
print(
f"file {alpha}: {filename}, dimensions: "
f"{DimensionsCheck[0]}, {DimensionsCheck[1]}, type: {myDataType[-1]}")
# process allBands option
allBandsIndex = None
allBandsCount = 1
if allBands:
if len(calc) > 1:
raise Exception("Error! --allBands implies a single --calc")
try:
allBandsIndex = myAlphaList.index(allBands)
except ValueError:
raise Exception(f"Error! allBands option was given but Band {allBands} not found. Cannot proceed")
allBandsCount = myFiles[allBandsIndex].RasterCount
if allBandsCount <= 1:
allBandsIndex = None
else:
allBandsCount = len(calc)
if extent not in [Extent.IGNORE, Extent.FAIL] and (
GeoTransformDiffer or isinstance(extent, GeoRectangle)):
# mixing different GeoTransforms/Extents
GeoTransformCheck, DimensionsCheck, ExtentCheck = extent_util.calc_geotransform_and_dimensions(
GeoTransforms, Dimensions, extent)
if GeoTransformCheck is None:
raise Exception("Error! The requested extent is empty. Cannot proceed")
for i in range(len(myFileNames)):
temp_vrt_filename, temp_vrt_ds = extent_util.make_temp_vrt(myFiles[i], ExtentCheck)
myTempFileNames.append(temp_vrt_filename)
myFiles[i] = None # close original ds
myFiles[i] = temp_vrt_ds # replace original ds with vrt_ds
# update the new precise dimensions and gt from the new ds
GeoTransformCheck = temp_vrt_ds.GetGeoTransform()
DimensionsCheck = [temp_vrt_ds.RasterXSize, temp_vrt_ds.RasterYSize]
temp_vrt_ds = None
################################################################
# set up output file
################################################################
# open output file exists
if outfile and os.path.isfile(outfile) and not overwrite:
if allBandsIndex is not None:
raise Exception("Error! allBands option was given but Output file exists, must use --overwrite option!")
if len(calc) > 1:
raise Exception(
"Error! multiple calc options were given but Output file exists, must use --overwrite option!")
if debug:
print(f"Output file {outfile} exists - filling in results into file")
myOut = open_ds(outfile, gdal.GA_Update)
if myOut is None:
error = 'but cannot be opened for update'
elif [myOut.RasterXSize, myOut.RasterYSize] != DimensionsCheck:
error = 'but is the wrong size'
elif ProjectionCheck and ProjectionCheck != myOut.GetProjection():
error = 'but is the wrong projection'
elif GeoTransformCheck and GeoTransformCheck != myOut.GetGeoTransform(can_return_null=True):
error = 'but is the wrong geotransform'
else:
error = None
if error:
raise Exception(
f"Error! Output exists, {error}. Use the --overwrite option "
f"to automatically overwrite the existing file")
myOutB = myOut.GetRasterBand(1)
myOutNDV = myOutB.GetNoDataValue()
myOutType = myOutB.DataType
else:
if outfile:
# remove existing file and regenerate
if os.path.isfile(outfile):
os.remove(outfile)
# create a new file
if debug:
print(f"Generating output file {outfile}")
else:
outfile = ''
# find data type to use
if not type:
# use the largest type of the input files
myOutType = max(myDataTypeNum)
else:
myOutType = type
if isinstance(myOutType, str):
myOutType = gdal.GetDataTypeByName(myOutType)
# create file
myOutDrv = gdal.GetDriverByName(format)
myOut = myOutDrv.Create(
os.fspath(outfile), DimensionsCheck[0], DimensionsCheck[1], allBandsCount,
myOutType, creation_options)
# set output geo info based on first input layer
if not GeoTransformCheck:
GeoTransformCheck = myFiles[0].GetGeoTransform(can_return_null=True)
if GeoTransformCheck:
myOut.SetGeoTransform(GeoTransformCheck)
if not ProjectionCheck:
ProjectionCheck = myFiles[0].GetProjection()
if ProjectionCheck:
myOut.SetProjection(ProjectionCheck)
if NoDataValue is None:
myOutNDV = None if hideNoData else DefaultNDVLookup[
myOutType] # use the default noDataValue for this datatype
elif isinstance(NoDataValue, str) and NoDataValue.lower() == 'none':
myOutNDV = None # not to set any noDataValue
else:
myOutNDV = NoDataValue # use the given noDataValue
for i in range(1, allBandsCount + 1):
myOutB = myOut.GetRasterBand(i)
if myOutNDV is not None:
myOutB.SetNoDataValue(myOutNDV)
if color_table:
# set color table and color interpretation
if is_path_like(color_table):
color_table = get_color_table(color_table)
myOutB.SetRasterColorTable(color_table)
myOutB.SetRasterColorInterpretation(gdal.GCI_PaletteIndex)
myOutB = None # write to band
myOutTypeName = gdal.GetDataTypeName(myOutType)
if debug:
print(f"output file: {outfile}, dimensions: {myOut.RasterXSize}, {myOut.RasterYSize}, type: {myOutTypeName}")
################################################################
# find block size to chop grids into bite-sized chunks
################################################################
# use the block size of the first layer to read efficiently
myBlockSize = myFiles[0].GetRasterBand(myBands[0]).GetBlockSize()
# find total x and y blocks to be read
nXBlocks = (int)((DimensionsCheck[0] + myBlockSize[0] - 1) / myBlockSize[0])
nYBlocks = (int)((DimensionsCheck[1] + myBlockSize[1] - 1) / myBlockSize[1])
myBufSize = myBlockSize[0] * myBlockSize[1]
if debug:
print(f"using blocksize {myBlockSize[0]} x {myBlockSize[1]}")
# variables for displaying progress
ProgressCt = -1
ProgressMk = -1
ProgressEnd = nXBlocks * nYBlocks * allBandsCount
################################################################
# start looping through each band in allBandsCount
################################################################
for bandNo in range(1, allBandsCount + 1):
################################################################
# start looping through blocks of data
################################################################
# store these numbers in variables that may change later
nXValid = myBlockSize[0]
nYValid = myBlockSize[1]
# loop through X-lines
for X in range(0, nXBlocks):
# in case the blocks don't fit perfectly
# change the block size of the final piece
if X == nXBlocks - 1:
nXValid = DimensionsCheck[0] - X * myBlockSize[0]
# find X offset
myX = X * myBlockSize[0]
# reset buffer size for start of Y loop
nYValid = myBlockSize[1]
myBufSize = nXValid * nYValid
# loop through Y lines
for Y in range(0, nYBlocks):
ProgressCt += 1
if 10 * ProgressCt / ProgressEnd % 10 != ProgressMk and not quiet:
ProgressMk = 10 * ProgressCt / ProgressEnd % 10
from sys import version_info
if version_info >= (3, 0, 0):
exec('print("%d.." % (10*ProgressMk), end=" ")')
else:
exec('print 10*ProgressMk, "..",')
# change the block size of the final piece
if Y == nYBlocks - 1:
nYValid = DimensionsCheck[1] - Y * myBlockSize[1]
myBufSize = nXValid * nYValid
# find Y offset
myY = Y * myBlockSize[1]
# create empty buffer to mark where nodata occurs
myNDVs = None
# make local namespace for calculation
local_namespace = {}
val_lists = defaultdict(list)
# fetch data for each input layer
for i, Alpha in enumerate(myAlphaList):
# populate lettered arrays with values
if allBandsIndex is not None and allBandsIndex == i:
myBandNo = bandNo
else:
myBandNo = myBands[i]
myval = gdal_array.BandReadAsArray(myFiles[i].GetRasterBand(myBandNo),
xoff=myX, yoff=myY,
win_xsize=nXValid, win_ysize=nYValid)
if myval is None:
raise Exception(f'Input block reading failed from filename {filename[i]}')
# fill in nodata values
if myNDV[i] is not None:
# myNDVs is a boolean buffer.
# a cell equals to 1 if there is NDV in any of the corresponding cells in input raster bands.
if myNDVs is None:
# this is the first band that has NDV set. we initializes myNDVs to a zero buffer
# as we didn't see any NDV value yet.
myNDVs = numpy.zeros(myBufSize)
myNDVs.shape = (nYValid, nXValid)
myNDVs = 1 * numpy.logical_or(myNDVs == 1, myval == myNDV[i])
# add an array of values for this block to the eval namespace
if Alpha in myAlphaFileLists:
val_lists[Alpha].append(myval)
else:
local_namespace[Alpha] = myval
myval = None
for lst in myAlphaFileLists:
local_namespace[lst] = val_lists[lst]
# try the calculation on the array blocks
this_calc = calc[bandNo - 1 if len(calc) > 1 else 0]
try:
myResult = eval(this_calc, global_namespace, local_namespace)
except:
print(f"evaluation of calculation {this_calc} failed")
raise
# Propagate nodata values (set nodata cells to zero
# then add nodata value to these cells).
if myNDVs is not None and myOutNDV is not None:
myResult = ((1 * (myNDVs == 0)) * myResult) + (myOutNDV * myNDVs)
elif not isinstance(myResult, numpy.ndarray):
myResult = numpy.ones((nYValid, nXValid)) * myResult
# write data block to the output file
myOutB = myOut.GetRasterBand(bandNo)
if gdal_array.BandWriteArray(myOutB, myResult, xoff=myX, yoff=myY) != 0:
raise Exception('Block writing failed')
myOutB = None # write to band
# remove temp files
for idx, tempFile in enumerate(myTempFileNames):
myFiles[idx] = None
os.remove(tempFile)
gdal.ErrorReset()
myOut.FlushCache()
if gdal.GetLastErrorMsg() != '':
raise Exception('Dataset writing failed')
if not quiet:
print("100 - Done")
return myOut
def reclassify_raster(infile,
outfile,
operator,
threshold,
pixel_value=100,
frmt="GTiff",
silence=True):
"""
Reclassify a raster into a single value according to a threshold
:param infile: path to the input
:param outfile: path to the output
:param operator: string, possible values ">","<",">=","<="
:param threshold: threshold value
:param pixel_value: the output pixel value for pixels that pass the threshold
:param frmt: the output format, default "GTiff"
:param silence: pass any value to enable debug info
:return: None
"""
'''using this instead of gdal_calc because gdal calc didn't work
scriptpath = os.path.join(os.path.dirname(__file__), '../', "pysdss/utility/gdal_calc.py")
scriptpath = os.path.normpath(scriptpath)
params = [scriptpath,"-A",path+input,"--outfile="+path+output,"--calc='(A>0.5)'", "--debug"]
print("Reclassify raster")
out,err = utils.run_script(params, callpy=["python3"])
print("output" + out)
if err:
print("ERROR:" + err)
raise Exception("gdal grid failed")'''
if operator not in [">", "<", ">=", "<="]:
raise Exception("unknown operator")
band1 = None
ds1 = None
bandOut = None
dsOut = None
try:
if not silence:
print("opening input")
# Open the dataset
ds1 = gdal.Open(infile, gdct.GA_ReadOnly)
band1 = ds1.GetRasterBand(1)
# Read the data into numpy arrays
data1 = gdar.BandReadAsArray(band1)
# get the nodata value
nodata = band1.GetNoDataValue()
if not silence:
print("filtering input")
# The actual calculation
filter = "data1" + operator + str(threshold)
data1[eval(filter)] = pixel_value
data1[data1 != pixel_value] = nodata
if not silence:
print("output result")
# Write the out file
driver = gdal.GetDriverByName(frmt)
dsOut = driver.Create(outfile, ds1.RasterXSize, ds1.RasterYSize, 1,
band1.DataType)
gdar.CopyDatasetInfo(ds1, dsOut)
bandOut = dsOut.GetRasterBand(1)
bandOut.SetNoDataValue(nodata)
gdar.BandWriteArray(bandOut, data1)
except RuntimeError as err:
raise err
except Exception as e:
raise e
finally:
# Close the datasets
if band1: band1 = None
if ds1: ds1 = None
if bandOut: bandOut = None
if dsOut: dsOut = None
