def NDSI(self, inFile_):
"""
Program description:
Normalized-Difference Snow Index (NDSI)
INPUT_PARAMETERS:
inputValue_ -
COMMENTS:
"""
try:
pB2 = inFile_.GetRasterBand(2)
pB5 = inFile_.GetRasterBand(5)
#Read the bands of the dataset into numpy arrays
pB2numpy = np.float32(gdalnumeric.BandReadAsArray(pB2))
pB5numpy = np.float32(gdalnumeric.BandReadAsArray(pB5))
#Calculation in numpy
pNum = np.subtract(pB2numpy, pB5numpy)
pDenom = np.add(pB2numpy, pB5numpy)
pNDSI = np.divide(pNum, pDenom)
except Exception:
raise Exception("Error: Error in calculating NDSI")
return pNDSI
def raster2array(raster, dim_ordering="channels_last", dtype='float32'):
'''
Modified from: https://gis.stackexchange.com/a/283207
'''
bands = [raster.GetRasterBand(i) for i in range(1, raster.RasterCount + 1)]
arr = np.array([gdn.BandReadAsArray(band) for band in bands]).astype(dtype)
return arr
def gdal_convert(inFolder, outFolder, inType, outType):
"""
Converts geotiffs and erdas imagine images to .tif,.jpg, .png, or .img
inputs:
inFolder (string): folder of .tif or .img images
outFolder (string): folder to save result to
inType (string): extension of input images ('.tif' or '.img')
outType (string): extension of output images ('.tif', '.img', '.jpg', '.png')
"""
for im in glob.glob(inFolder + '/*' + inType):
print('in: ' + im)
imName = os.path.splitext(os.path.basename(im))[0]
outIm = os.path.join(outFolder, imName + outType)
print('out: ' + outIm)
if outType == '.npy':
raster = gdal.Open(im)
bands = [
raster.GetRasterBand(i)
for i in range(1, raster.RasterCount + 1)
]
arr = np.array([gdn.BandReadAsArray(band)
for band in bands]).astype('float32')
arr = np.transpose(arr, [1, 2, 0])
np.save(outIm, arr)
else:
raster = gdal.Open(im)
gdal.Translate(outIm, raster)
raster = None
def read_tif(tif, band=1, nodata=0):
"""reads in the a tif, the specified band or all bands
band default is 1
if you read in on band the return is a 2d array
if band is set to 'all', it will return a 3d array (bands, x, y)"""
try:
#default band is 1 and default for return nodata value is False ~ 0 ;1 ~ True
inTif = zz_gdalnum.gdal.Open(tif, zz_gdalcon.GA_ReadOnly)
if type(inTif) != 'NoneType':
#test if passed band is in the raster
nr_of_bands = inTif.RasterCount
if band > nr_of_bands and band != 'all':
raise NameError(
'band not in file max Nr. of bands: {0}'.format(
nr_of_bands))
if band == 'all':
#create a 3d stack of the data (band, x, y)
bands = inTif.GetRasterBand(1)
data = zz_gdalnum.BandReadAsArray(bands)
y, x = data.shape
shape = (1, y, x)
data = data.reshape(shape)
for b in range(2, nr_of_bands):
bands = inTif.GetRasterBand(b)
data = np.vstack(
(data,
zz_gdalnum.BandReadAsArray(bands).reshape(shape)))
else:
band = inTif.GetRasterBand(band)
data = zz_gdalnum.BandReadAsArray(band)
inTif = None
if type(data) == None.__class__:
raise
else:
if nodata == 0:
return data
elif nodata == 1:
noda = band.GetNoDataValue()
return data, noda
else:
raise NameError('input is not a file or file is broken')
except:
print "Error:", sys.exc_info()[:2]
inTif = None
raise
def raster_2_array(raster_full_path):
"""Read a raster dem as array
"""
data_set = gdal.Open(raster_full_path)
data_set_band = data_set.GetRasterBand(1)
raster_array = gnum.BandReadAsArray(data_set_band)
no_data_value = data_set_band.GetNoDataValue()
return raster_array, no_data_value
def img_to_array(input_file, dim_ordering="channels_last", dtype='uint32'):
#https://gis.stackexchange.com/questions/32995/fully-load-raster-into-a-numpy-array/33070
file = gdal.Open(input_file, gdal.GA_ReadOnly)
bands = [file.GetRasterBand(i) for i in range(1, file.RasterCount + 1)]
arr = np.array([gdn.BandReadAsArray(band) for band in bands]).astype(dtype)
if dim_ordering == "channels_last":
arr = np.transpose(
arr, [1, 2, 0]) # Reorders dimensions, so that channels are last
return arr
def img_to_array(tif_path, dtype='float32'):
"""
Returns numpy array of the input tif_path.
"""
file = gdal.Open(tif_path)
bands = [file.GetRasterBand(i) for i in range(1, file.RasterCount + 1)]
arr = np.array([gdn.BandReadAsArray(band) for band in bands]).astype(dtype)
arr = np.transpose(
arr, [1, 2, 0]) # Reordering dimensions, so that channels are last
return arr
def isothermal():
alpha = -0.0055182680606
beta = 28.2088488402
raster_filename = '../data/SRTM_Bog_DEM.tif'
raster = gdal.Open(raster_filename)
trans_data = raster.GetGeoTransform()
band = raster.GetRasterBand(1)
no_data_value = band.GetNoDataValue()
output = gdalnumeric.BandReadAsArray(band) * alpha + beta
pd.DataFrame(output).to_clipboard()
pass
def read_data(inTif, band_nr, nodata=0):
band = inTif.GetRasterBand(band_nr)
data = zz_gdalnum.BandReadAsArray(band)
inTif = None
if type(data)==None.__class__:
raise
else:
if nodata==0:
return data
elif nodata==1:
noda = band.GetNoDataValue()
return data, noda
def img_to_array(input_file, dtype='uint16'): ###reads multiband image as ndarray
#specify band list - indices of each band# you want to open as multiband array
#not specifying band list opens all bands into an array
#returns wavelengths only when header is available for image
file = gdal.Open(input_file)
bands = [file.GetRasterBand(i) for i in range(1, file.RasterCount + 1)]
arr = np.array([gdn.BandReadAsArray(band) for band in bands]).astype(dtype)
#geotransform = file.GetGeoTransform()
#spatialreference = file.GetProjection()
#sometimes single band images have shape of length 2- reshape to (bands, rows, cols) to be consistent
if len(arr.shape) == 2:
arr = np.reshape(arr, (1, arr.shape[0], arr.shape[1]))
return arr
def get_raster_attribute(self, attribute, **kwargs):
raster = gdal.DEMProcessing('',
self.file,
attribute,
format='MEM',
**kwargs)
raster_band = raster.GetRasterBand(1)
raster_values = np.ma.masked_values(
gdalnumeric.BandReadAsArray(raster_band),
raster_band.GetNoDataValue(),
copy=False)
del raster
del raster_band
return raster_values
def copy_into(self, output_file, source_band=1, target_band=1):
"""
Copy data to target file and filter according to the type given in
the initializer.
output_file -- gdal.Dataset object for the file into which some or all
of this file may be copied.
"""
t_geotransform = output_file.GetGeoTransform()
t_ulx = t_geotransform[0]
t_uly = t_geotransform[3]
t_pixel_width = t_geotransform[1]
t_pixel_height = t_geotransform[5]
# Intersection region
tgw_ulx = max(t_ulx, self.ulx)
if t_pixel_height < 0:
tgw_uly = min(t_uly, self.uly)
else:
tgw_uly = max(t_uly, self.uly)
# Target window in pixel coordinates.
tw_xoff = int((tgw_ulx - t_ulx) / t_pixel_width + 0.1)
tw_yoff = int((tgw_uly - t_uly) / t_pixel_height + 0.1)
source_file = gdal.Open(self.filename, gdalconst.GA_ReadOnly)
target_band = output_file.GetRasterBand(target_band)
source_values = gdalnumeric.BandReadAsArray(
source_file.GetRasterBand(source_band),
buf_type=gdal.GDT_Float32
)
no_data_value = target_band.GetNoDataValue()
# Filter by upper and lower band threshold values
source_values[source_values < self.lower_limit] = no_data_value
source_values[source_values > self.upper_limit] = no_data_value
gdalnumeric.BandWriteArray(
target_band, source_values, xoff=tw_xoff, yoff=tw_yoff
)
del source_values
del source_file
def __copy_band_data(self):
target_band = self.output_file.GetRasterBand(self.BAND_NUMBER)
target_band.SetNoDataValue(self.BAND_NO_DATA_VALUE)
source_values = gdalnumeric.BandReadAsArray(
self.mosaic_vrt.GetRasterBand(self.BAND_NUMBER),
buf_type=self.BAND_DATA_TYPE,
)
# Filter by upper and lower band threshold values
source_values[(source_values < self.lower_limit) | (
source_values > self.upper_limit)] = self.BAND_NO_DATA_VALUE
gdalnumeric.BandWriteArray(target_band, source_values)
target_band.FlushCache()
target_band.SetMetadata(self.BAND_METADATA[self.source_type])
del target_band
del source_values
def img_to_array(input_file, dim_ordering="channels_last", dtype='float32'):
"""
Loads multi-spectral raster to numpy array using gdal.
Based on: https://gis.stackexchange.com/questions/32995/fully-load-raster-into-a-numpy-array
Parameters:
input_file (tiff): multi-spectral raster
dim_ordering (string): reorders dimensions, so that channels are last
Returns:
arr (ndarray): 2D array
"""
file = gdal.Open(input_file)
bands = [file.GetRasterBand(i) for i in range(1, file.RasterCount + 1)]
arr = np.array([gdn.BandReadAsArray(band) for band in bands])
if dim_ordering=="channels_last":
arr = np.transpose(arr, [1, 2, 0]) # Reorders dimensions, so that channels are last
return arr
def img_to_array(input_file, dim_ordering="HWC", dtype='float32', band_mapping=None, return_extent=False):
"""Reads an image from disk and returns it as numpy array.
Args:
input_file: Path to the input file.
dim_ordering: One of HWC or CHW, C=Channels, H=Height, W=Width
dtype: Desired data type for loading, e.g. np.uint8, np.float32...
band_mapping: Dictionary of which image band to load into which array band. E.g. {1:0, 3:1}
return_extent: Whether or not to return the raster extent in the form (ymin, ymax, xmin, xmax). Defaults to False.
Returns:
Numpy array containing the image and optionally the extent.
"""
ds = gdal.Open(input_file)
extent = get_map_extent(ds)
if band_mapping is None:
num_bands = ds.RasterCount
band_mapping = {i+1: i for i in range(num_bands)}
elif isinstance(band_mapping, dict):
num_bands = len(band_mapping)
else:
raise TypeError("band_mapping must be a dict, not {}.".format(type(band_mapping)))
arr = np.empty((num_bands, ds.RasterYSize, ds.RasterXSize), dtype=dtype)
for source_layer, target_layer in band_mapping.items():
arr[target_layer] = gdn.BandReadAsArray(ds.GetRasterBand(source_layer))
if dim_ordering == "HWC":
arr = np.transpose(arr, (1, 2, 0)) # Reorders dimensions, so that channels are last
elif dim_ordering == "CHW":
pass
else:
raise ValueError("Dim ordering {} not supported. Choose one of 'HWC' or 'CHW'.".format(dim_ordering))
if return_extent:
return arr, extent
else:
return arr
def __init__(self, source_folder, source_type):
self.filename = self.get_file_name(source_type)
file = gdal.Open(os.path.join(source_folder, self.filename))
self.bands = file.RasterCount
_raster_band = file.GetRasterBand(1)
self.band_type = _raster_band.DataType
self.no_data_value = _raster_band.GetNoDataValue()
self.band_values = gdalnumeric.BandReadAsArray(_raster_band)
_projection = osr.SpatialReference(wkt=file.GetProjection())
self.projection = _projection.GetAttrValue('geogcs')
self.xsize = file.RasterXSize
self.ysize = file.RasterYSize
_geo_transform = file.GetGeoTransform()
self.ulx = _geo_transform[0]
self.uly = _geo_transform[3]
self.pixel_width = _geo_transform[1]
self.pixel_height = _geo_transform[5]
del file
def band_values(self, **kwargs):
"""
Method to read band from arguments or from initialized raster.
Will mask values defined in the band NoDataValue and store this mask
with the `current_mask` property if the band is the same as the
initialized one.
:param kwargs:
'band_number': band_number to read instead of the one given with
the initialize call.
:return: Numpy masked array
"""
band_number = kwargs.get('band_number', self.band_number)
band = self.file.GetRasterBand(band_number)
values = np.ma.masked_values(gdalnumeric.BandReadAsArray(band),
band.GetNoDataValue(),
copy=False)
del band
return values
def run(self):
self._abort = False
self.progress.emit("Starting RasterBender", float(0), float(0))
#####################################
# Step 1 : create the delaunay mesh #
#####################################
self.progress.emit("Loading delaunay mesh...", float(0), float(0))
# Create the delaunay triangulation
triangles, pointsA, pointsB, hull, constraints = triangulate.triangulate(
self.pairsLayer, self.pairsLimitToSelection, self.constraintsLayer,
self.constraintsLimitToSelection, self.bufferValue)
###############################
# Step 2. Opening the dataset #
###############################
self.progress.emit(
"Opening the dataset... This shouldn't be too long...", float(0),
float(0))
#Open the dataset
gdal.UseExceptions()
# Read the source data into numpy arrays
dsSource = gdal.Open(self.sourcePath, gdal.GA_ReadOnly)
sourceDataR = gdalnumeric.BandReadAsArray(dsSource.GetRasterBand(1))
sourceDataG = gdalnumeric.BandReadAsArray(dsSource.GetRasterBand(2))
sourceDataB = gdalnumeric.BandReadAsArray(dsSource.GetRasterBand(3))
# Get the transformation
pixW = float(dsSource.RasterXSize - 1) #width in pixel
pixH = float(dsSource.RasterYSize - 1) #width in pixel
mapW = float(dsSource.RasterXSize) * dsSource.GetGeoTransform()[
1] #width in map units
mapH = float(dsSource.RasterYSize) * dsSource.GetGeoTransform()[
5] #width in map units
offX = dsSource.GetGeoTransform()[0] #offset in map units
offY = dsSource.GetGeoTransform()[3] #offset in map units
# Open the target into numpy array
#dsTarget = gdal.Open(self.targetPath, gdal.GA_Update )
driver = gdal.GetDriverByName("GTiff")
dsTarget = driver.CreateCopy(self.targetPath, dsSource, 0)
#dsTarget.SetGeoTransform( dsSource.GetGeoTransform() )
dsTarget = None #close
def xyToQgsPoint(x, y):
return QgsPoint(offX + mapW * (x / pixW), offY + mapH * (y / pixH))
def qgsPointToXY(qgspoint):
return (int((qgspoint.x() - offX) / mapW * pixW),
int((qgspoint.y() - offY) / mapH * pixH))
#######################################
# Step 3A. Looping through the blocks #
#######################################
#Loop through every block
blockCountX = dsSource.RasterXSize // self.blockSize + 1
blockCountY = dsSource.RasterYSize // self.blockSize + 1
blockCount = blockCountX * blockCountY
blockI = 0
displayTotal = dsSource.RasterXSize * dsSource.RasterYSize
displayStep = min((self.blockSize**2) / 20,
10000) # update gui every n steps
self.progress.emit("Starting computation... This can take a while...",
float(0), float(0))
for blockNumY in range(0, blockCountY):
blockOffsetY = blockNumY * self.blockSize
blockH = min(self.blockSize, dsSource.RasterYSize - blockOffsetY)
if blockH <= 0: continue
for blockNumX in range(0, blockCountX):
blockOffsetX = blockNumX * self.blockSize
blockW = min(self.blockSize,
dsSource.RasterXSize - blockOffsetX)
if blockW <= 0: continue
blockI += 1
pixelCount = blockW * blockH
pixelI = 0
blockRectangle = QgsRectangle(
xyToQgsPoint(blockOffsetX, blockOffsetY),
xyToQgsPoint(blockOffsetX + blockW, blockOffsetY + blockH)
) # this is the shape of the block, used for optimization
# We check if the block intersects the hull, if not, we skip it
if not hull.intersects(blockRectangle):
self.progress.emit(
"Block %i out of %i is out of the convex hull, we skip it..."
% (blockI, blockCount), float(0),
float(blockI / float(blockCount)))
continue
# We create the trifinder for the block
blockTrifinder = trifinder.Trifinder(pointsB, triangles,
blockRectangle)
targetDataR = gdalnumeric.BandReadAsArray(
dsSource.GetRasterBand(1), blockOffsetX, blockOffsetY,
blockW, blockH)
targetDataG = gdalnumeric.BandReadAsArray(
dsSource.GetRasterBand(2), blockOffsetX, blockOffsetY,
blockW, blockH)
targetDataB = gdalnumeric.BandReadAsArray(
dsSource.GetRasterBand(3), blockOffsetX, blockOffsetY,
blockW, blockH)
#######################################
# Step 3B. Looping through the pixels #
#######################################
# Loop through every pixel
for y in range(0, blockH):
for x in range(0, blockW):
# If abort was called, we finish the process
if self._abort:
self.error.emit(
"Aborted on pixel %i out of %i on block %i out of %i..."
% (pixelI, pixelCount, blockI, blockCount),
float(0), float(0))
return
pixelI += 1
# Ever now and then, we update the status
if pixelI % displayStep == 0:
self.progress.emit(
"Working on pixel %i out of %i on block %i out of %i... Trifinder has %i triangles"
% (pixelI, pixelCount, blockI, blockCount,
len(blockTrifinder.triangles)),
float(pixelI) / float(pixelCount),
float(blockI) / float(blockCount))
# We find in which triangle the point lies using the trifinder.
p = xyToQgsPoint(blockOffsetX + x, blockOffsetY + y)
tri = blockTrifinder.find(p)
if tri is None:
# If it's in no triangle, we don't change it
continue
# If it's in a triangle, we transform the coordinates
newP = trimapper.map(p, pointsB[tri[0]],
pointsB[tri[1]], pointsB[tri[2]],
pointsA[tri[0]], pointsA[tri[1]],
pointsA[tri[2]])
newX, newY = qgsPointToXY(newP)
# TODO : this would maybe get interpolated results
#ident = sourceRaster.dataProvider().identify( pt, QgsRaster.IdentifyFormatValue)
#targetDataR[y][x] = ident.results()[1]
#targetDataG[y][x] = ident.results()[2]
#targetDataB[y][x] = ident.results()[3]
try:
if newY < 0 or newX < 0:
raise IndexError() #avoid looping
targetDataR[y][x] = sourceDataR[newY][newX]
targetDataG[y][x] = sourceDataG[newY][newX]
targetDataB[y][x] = sourceDataB[newY][newX]
except IndexError, e:
targetDataR[y][x] = 0
targetDataG[y][x] = 0
targetDataB[y][x] = 0
# Write to the image
dsTarget = gdal.Open(self.targetPath, gdal.GA_Update)
gdalnumeric.BandWriteArray(dsTarget.GetRasterBand(1),
targetDataR, blockOffsetX,
blockOffsetY)
gdalnumeric.BandWriteArray(dsTarget.GetRasterBand(2),
targetDataG, blockOffsetX,
blockOffsetY)
gdalnumeric.BandWriteArray(dsTarget.GetRasterBand(3),
targetDataB, blockOffsetX,
blockOffsetY)
dsTarget = None
def createDistanceTransform(rasterSrc,
vectorSrc,
npDistFileName='',
units='pixels'):
## open source vector file that truth data
source_ds = ogr.Open(vectorSrc)
source_layer = source_ds.GetLayer()
## extract data from src Raster File to be emulated
## open raster file that is to be emulated
srcRas_ds = gdal.Open(rasterSrc)
cols = srcRas_ds.RasterXSize
rows = srcRas_ds.RasterYSize
noDataValue = 0
if units == 'meters':
geoTrans, poly, ulX, ulY, lrX, lrY = gT.getRasterExtent(srcRas_ds)
transform_WGS84_To_UTM, transform_UTM_To_WGS84, utm_cs = gT.createUTMTransform(
poly)
line = ogr.Geometry(ogr.wkbLineString)
line.AddPoint(geoTrans[0], geoTrans[3])
line.AddPoint(geoTrans[0] + geoTrans[1], geoTrans[3])
line.Transform(transform_WGS84_To_UTM)
metersIndex = line.Length()
else:
metersIndex = 1
## create First raster memory layer
memdrv = gdal.GetDriverByName('MEM')
dst_ds = memdrv.Create('', cols, rows, 1, gdal.GDT_Byte)
dst_ds.SetGeoTransform(srcRas_ds.GetGeoTransform())
dst_ds.SetProjection(srcRas_ds.GetProjection())
band = dst_ds.GetRasterBand(1)
band.SetNoDataValue(noDataValue)
gdal.RasterizeLayer(dst_ds, [1], source_layer, burn_values=[255])
srcBand = dst_ds.GetRasterBand(1)
memdrv2 = gdal.GetDriverByName('MEM')
prox_ds = memdrv2.Create('', cols, rows, 1, gdal.GDT_Int16)
prox_ds.SetGeoTransform(srcRas_ds.GetGeoTransform())
prox_ds.SetProjection(srcRas_ds.GetProjection())
proxBand = prox_ds.GetRasterBand(1)
proxBand.SetNoDataValue(noDataValue)
options = ['NODATA=0']
##compute distance to non-zero pixel values and scrBand and store in proxBand
gdal.ComputeProximity(srcBand, proxBand, options)
memdrv3 = gdal.GetDriverByName('MEM')
proxIn_ds = memdrv3.Create('', cols, rows, 1, gdal.GDT_Int16)
proxIn_ds.SetGeoTransform(srcRas_ds.GetGeoTransform())
proxIn_ds.SetProjection(srcRas_ds.GetProjection())
proxInBand = proxIn_ds.GetRasterBand(1)
proxInBand.SetNoDataValue(noDataValue)
options = ['NODATA=0', 'VALUES=0']
##compute distance to zero pixel values and scrBand and store in proxInBand
gdal.ComputeProximity(srcBand, proxInBand, options)
proxIn = gdalnumeric.BandReadAsArray(proxInBand)
proxOut = gdalnumeric.BandReadAsArray(proxBand)
##distance tranform is the distance to zero pixel values minus distance to non-zero pixel values
proxTotal = proxIn.astype(float) - proxOut.astype(float)
proxTotal = proxTotal * metersIndex
if npDistFileName != '':
np.save(npDistFileName, proxTotal)
return proxTotal
def createDistanceTransformByFeatureIndex(feature_index,
rasterSrc,
vectorSrc,
npDistFileName='',
units='pixels'):
## open source vector file that truth data
source_ds = ogr.Open(vectorSrc)
source_layer = source_ds.GetLayer()
#Define feature
my_feature = source_layer[feature_index]
#Spatial Reference
srs = source_layer.GetSpatialRef()
#Create feature Layer
outDriver = ogr.GetDriverByName('MEMORY')
outDataSource = outDriver.CreateDataSource('memData')
Feature_Layer = outDataSource.CreateLayer("this_feature",
srs,
geom_type=ogr.wkbPolygon)
#Add feature to layer
Feature_Layer.CreateFeature(my_feature)
## extract data from src Raster File to be emulated
## open raster file that is to be emulated
srcRas_ds = gdal.Open(rasterSrc)
cols = srcRas_ds.RasterXSize
rows = srcRas_ds.RasterYSize
noDataValue = 0
metersIndex = 1
## create First raster memory layer
memdrv = gdal.GetDriverByName('MEM')
dst_ds = memdrv.Create('', cols, rows, 1, gdal.GDT_Byte)
dst_ds.SetGeoTransform(srcRas_ds.GetGeoTransform())
dst_ds.SetProjection(srcRas_ds.GetProjection())
band = dst_ds.GetRasterBand(1)
band.SetNoDataValue(noDataValue)
gdal.RasterizeLayer(dst_ds, [1], Feature_Layer, burn_values=[255])
srcBand = dst_ds.GetRasterBand(1)
memdrv2 = gdal.GetDriverByName('MEM')
prox_ds = memdrv2.Create('', cols, rows, 1, gdal.GDT_Int16)
prox_ds.SetGeoTransform(srcRas_ds.GetGeoTransform())
prox_ds.SetProjection(srcRas_ds.GetProjection())
proxBand = prox_ds.GetRasterBand(1)
proxBand.SetNoDataValue(noDataValue)
options = ['NODATA=0']
gdal.ComputeProximity(srcBand, proxBand, options)
memdrv3 = gdal.GetDriverByName('MEM')
proxIn_ds = memdrv3.Create('', cols, rows, 1, gdal.GDT_Int16)
proxIn_ds.SetGeoTransform(srcRas_ds.GetGeoTransform())
proxIn_ds.SetProjection(srcRas_ds.GetProjection())
proxInBand = proxIn_ds.GetRasterBand(1)
proxInBand.SetNoDataValue(noDataValue)
options = ['NODATA=0', 'VALUES=0']
gdal.ComputeProximity(srcBand, proxInBand, options)
proxIn = gdalnumeric.BandReadAsArray(proxInBand)
proxOut = gdalnumeric.BandReadAsArray(proxBand)
proxTotal = proxIn.astype(float) - proxOut.astype(float)
proxTotal = proxTotal * metersIndex
if npDistFileName != '':
np.save(npDistFileName, proxTotal)
return proxTotal
if (tilesy*ybs!=ysize): tilesy=tilesy+1
t1=time.time()
tread[nthread-1]+=t1-t0
for itx in range(0,tilesx*xbs,xbs):
for ity in range(0,tilesy*ybs,ybs):
_xbs=xbs if itx+xbs<xsize else xsize-itx
_ybs=ybs if ity+ybs<ysize else ysize-ity
t2=time.time()
red_array = gdalnumeric.BandReadAsArray(red_band,
xoff=itx,
yoff=ity,
win_xsize=_xbs,
win_ysize=_ybs,
buf_type=gdalconst.GDT_UInt16)
t3=time.time()
ndvi_array = red_array*0.5
ndvi_band.WriteArray(ndvi_array, xoff=itx, yoff=ity)
ndvi_array = None
t4=time.time()
tread[nthread-1]+=t3-t2
tprocess[nthread-1]+=t4-t3
def create_dist_map(rasterSrc,
vectorSrc,
npDistFileName='',
noDataValue=0,
burn_values=1,
dist_mult=1,
vmax_dist=64):
'''
Create building signed distance transform from Yuan 2016
(https://arxiv.org/pdf/1602.06564v1.pdf).
vmax_dist: absolute value of maximum distance (meters) from building edge
Adapted from createNPPixArray in labeltools
'''
## open source vector file that truth data
source_ds = ogr.Open(vectorSrc)
source_layer = source_ds.GetLayer()
## extract data from src Raster File to be emulated
## open raster file that is to be emulated
srcRas_ds = gdal.Open(rasterSrc)
cols = srcRas_ds.RasterXSize
rows = srcRas_ds.RasterYSize
geoTrans, poly, ulX, ulY, lrX, lrY = gT.getRasterExtent(srcRas_ds)
transform_WGS84_To_UTM, transform_UTM_To_WGS84, utm_cs \
= gT.createUTMTransform(poly)
line = ogr.Geometry(ogr.wkbLineString)
line.AddPoint(geoTrans[0], geoTrans[3])
line.AddPoint(geoTrans[0] + geoTrans[1], geoTrans[3])
line.Transform(transform_WGS84_To_UTM)
metersIndex = line.Length()
memdrv = gdal.GetDriverByName('MEM')
dst_ds = memdrv.Create('', cols, rows, 1, gdal.GDT_Byte)
dst_ds.SetGeoTransform(srcRas_ds.GetGeoTransform())
dst_ds.SetProjection(srcRas_ds.GetProjection())
band = dst_ds.GetRasterBand(1)
band.SetNoDataValue(noDataValue)
gdal.RasterizeLayer(dst_ds, [1], source_layer, burn_values=[burn_values])
srcBand = dst_ds.GetRasterBand(1)
memdrv2 = gdal.GetDriverByName('MEM')
prox_ds = memdrv2.Create('', cols, rows, 1, gdal.GDT_Int16)
prox_ds.SetGeoTransform(srcRas_ds.GetGeoTransform())
prox_ds.SetProjection(srcRas_ds.GetProjection())
proxBand = prox_ds.GetRasterBand(1)
proxBand.SetNoDataValue(noDataValue)
opt_string = 'NODATA=' + str(noDataValue)
options = [opt_string]
gdal.ComputeProximity(srcBand, proxBand, options)
memdrv3 = gdal.GetDriverByName('MEM')
proxIn_ds = memdrv3.Create('', cols, rows, 1, gdal.GDT_Int16)
proxIn_ds.SetGeoTransform(srcRas_ds.GetGeoTransform())
proxIn_ds.SetProjection(srcRas_ds.GetProjection())
proxInBand = proxIn_ds.GetRasterBand(1)
proxInBand.SetNoDataValue(noDataValue)
opt_string2 = 'VALUES=' + str(noDataValue)
options = [opt_string, opt_string2]
#options = ['NODATA=0', 'VALUES=0']
gdal.ComputeProximity(srcBand, proxInBand, options)
proxIn = gdalnumeric.BandReadAsArray(proxInBand)
proxOut = gdalnumeric.BandReadAsArray(proxBand)
proxTotal = proxIn.astype(float) - proxOut.astype(float)
proxTotal = proxTotal * metersIndex
proxTotal *= dist_mult
# clip array
proxTotal = np.clip(proxTotal, -1 * vmax_dist, 1 * vmax_dist)
if npDistFileName != '':
# save as numpy file since some values will be negative
np.save(npDistFileName, proxTotal)
#cv2.imwrite(npDistFileName, proxTotal)
return proxTotal
def extract(raster_in, mask, class_nb, class_names, out_file_pickle,
out_file_csv):
"""
Extract values from raster file. Saves into csv and pickle file.
raster_in: in raster to extract values
mask: mask with rasterised training data
class_nb: number of classes to extract values
class_names: class names to extract values
out_file_csv: output csv
out_file_pickle: output pickle
"""
drivers_raster = gdal.GetDriverByName('ENVI')
drivers_raster.Register()
raster = gdal.Open(raster_in, gdalconst.GA_ReadOnly)
inmask = gdal.Open(mask, gdalconst.GA_ReadOnly)
band_mask = inmask.GetRasterBand(2)
data_mask = band_mask.ReadAsArray(0, 0)
coords = np.nonzero(data_mask)
new_coords = np.array([0, 0])
for i in range(len(coords[0])): # reads coordinates from input raster
m = np.array([coords[0][i], coords[1][i]])
new_coords = np.vstack((new_coords, m))
np.delete(new_coords, 0, 0) # removers first empty row
pixel_class = ([data_mask[x, y] for x, y in new_coords])
px_vals = [[] for x in range(class_nb)]
for nb, x in enumerate(pixel_class):
px_vals[x - 1].append(new_coords[nb])
data = []
band_mask_index = inmask.GetRasterBand(1)
for nb, class_nb in enumerate(px_vals):
coord_list_class = px_vals[nb]
class_id = nb + 1
for counter, i in enumerate(coord_list_class):
x, y = int(i[0]), int(i[1])
bands = [
raster.GetRasterBand(i)
for i in range(1, raster.RasterCount + 1)
]
pix_val = np.squeeze(
np.array([
gdalnumeric.BandReadAsArray(band, y, x, 1, 1)
for band in bands
]).astype('int64'))
pixel_extract = [x] + [y] + [pix_val] + ['{0}'.format(class_names[class_id])] + \
[int(band_mask_index.ReadAsArray(y, x, 1, 1))]
data.append(pixel_extract)
print('extracted', round((counter + 1) / len(coord_list_class), 2),
'% form class {0}'.format(class_names[class_id]))
# cleaning data
x = [x[0] for x in data]
y = [x[1] for x in data]
values = [x[2] for x in data]
class_name = [x[3] for x in data]
index = [x[4] for x in data]
df = pd.DataFrame(
list(zip(x, y, values, class_name, index)),
columns=['x', 'y', 'coordinates', 'values', 'class', 'index'])
print(df.loc[:, 'class'].value_counts())
df.to_csv(out_file_csv)
df.to_pickle(out_file_pickle)
print('done!')
def create_dist_map(rasterSrc, vectorSrc, npDistFileName='',
noDataValue=0, burn_values=255,
dist_mult=1, vmax_dist=64):
source_ds = ogr.Open(vectorSrc)
source_layer = source_ds.GetLayer()
srcRas_ds = gdal.Open(rasterSrc)
cols = srcRas_ds.RasterXSize
rows = srcRas_ds.RasterYSize
geoTrans, poly, ulX, ulY, lrX, lrY = gT.getRasterExtent(srcRas_ds)
transform_WGS84_To_UTM, transform_UTM_To_WGS84, utm_cs \
= gT.createUTMTransform(poly)
line = ogr.Geometry(ogr.wkbLineString)
line.AddPoint(geoTrans[0], geoTrans[3])
line.AddPoint(geoTrans[0]+geoTrans[1], geoTrans[3])
line.Transform(transform_WGS84_To_UTM)
metersIndex = line.Length()
memdrv = gdal.GetDriverByName('MEM')
dst_ds = memdrv.Create('', cols, rows, 1, gdal.GDT_Byte)
dst_ds.SetGeoTransform(srcRas_ds.GetGeoTransform())
dst_ds.SetProjection(srcRas_ds.GetProjection())
band = dst_ds.GetRasterBand(1)
band.SetNoDataValue(noDataValue)
gdal.RasterizeLayer(dst_ds, [1], source_layer, burn_values=[burn_values])
srcBand = dst_ds.GetRasterBand(1)
memdrv2 = gdal.GetDriverByName('MEM')
prox_ds = memdrv2.Create('', cols, rows, 1, gdal.GDT_Int16)
prox_ds.SetGeoTransform(srcRas_ds.GetGeoTransform())
prox_ds.SetProjection(srcRas_ds.GetProjection())
proxBand = prox_ds.GetRasterBand(1)
proxBand.SetNoDataValue(noDataValue)
opt_string = 'NODATA='+str(noDataValue)
options = [opt_string]
gdal.ComputeProximity(srcBand, proxBand, options)
memdrv3 = gdal.GetDriverByName('MEM')
proxIn_ds = memdrv3.Create('', cols, rows, 1, gdal.GDT_Int16)
proxIn_ds.SetGeoTransform(srcRas_ds.GetGeoTransform())
proxIn_ds.SetProjection(srcRas_ds.GetProjection())
proxInBand = proxIn_ds.GetRasterBand(1)
proxInBand.SetNoDataValue(noDataValue)
opt_string2 = 'VALUES='+str(noDataValue)
options = [opt_string, opt_string2]
gdal.ComputeProximity(srcBand, proxInBand, options)
proxIn = gdalnumeric.BandReadAsArray(proxInBand)
proxOut = gdalnumeric.BandReadAsArray(proxBand)
proxTotal = proxIn.astype(float) - proxOut.astype(float)
proxTotal = proxTotal*metersIndex
proxTotal *= dist_mult
proxTotal = np.clip(proxTotal, -1*vmax_dist, 1*vmax_dist)
if npDistFileName != '':
np.save(npDistFileName, proxTotal)
def img_to_array(input_file, dim_ordering="channels_last", dtype='float32':
"""
Loads multi-spectral raster to numpy array using gdal.
Parameters:
input_file (tiff): multi-spectral raster
dim_ordering (string): reorders dimensions, so that channels are last
Returns:
arr (ndarray): 2D array
"""
file = gdal.Open(input_file)
bands = [file.GetRasterBand(i) for i in range(1, file.RasterCount + 1)]
arr = np.array([gdn.BandReadAsArray(band) for band in bands])
if dim_ordering=="channels_last":
arr = np.transpose(arr, [1, 2, 0]) # Reorders dimensions, so that channels are last
return arr
def labels_to_array(input_file):
""" Loads geotif labels to numpy array """
labels = Image.open(input_file)
arr = np.array(labels)
return arr
def doit(opts, args):
# pylint: disable=unused-argument
if opts.debug:
print("gdal_calc.py starting calculation %s" % (opts.calc))
# set up global namespace for eval with all functions of gdalnumeric
global_namespace = dict([(key, getattr(gdalnumeric, key))
for key in dir(gdalnumeric) if not key.startswith('__')])
if not opts.calc:
raise Exception("No calculation provided.")
elif not opts.outF:
raise Exception("No output file provided.")
if opts.format is None:
opts.format = GetOutputDriverFor(opts.outF)
################################################################
# fetch details of input layers
################################################################
# set up some lists to store data for each band
myFiles = []
myBands = []
myAlphaList = []
myDataType = []
myDataTypeNum = []
myNDV = []
DimensionsCheck = None
# loop through input files - checking dimensions
for myI, myF in opts.input_files.items():
if not myI.endswith("_band"):
# check if we have asked for a specific band...
if "%s_band" % myI in opts.input_files:
myBand = opts.input_files["%s_band" % myI]
else:
myBand = 1
myFile = gdal.Open(myF, gdal.GA_ReadOnly)
if not myFile:
raise IOError("No such file or directory: '%s'" % myF)
myFiles.append(myFile)
myBands.append(myBand)
myAlphaList.append(myI)
myDataType.append(gdal.GetDataTypeName(myFile.GetRasterBand(myBand).DataType))
myDataTypeNum.append(myFile.GetRasterBand(myBand).DataType)
myNDV.append(myFile.GetRasterBand(myBand).GetNoDataValue())
# check that the dimensions of each layer are the same
if DimensionsCheck:
if DimensionsCheck != [myFile.RasterXSize, myFile.RasterYSize]:
raise Exception("Error! Dimensions of file %s (%i, %i) are different from other files (%i, %i). Cannot proceed" %
(myF, myFile.RasterXSize, myFile.RasterYSize, DimensionsCheck[0], DimensionsCheck[1]))
else:
DimensionsCheck = [myFile.RasterXSize, myFile.RasterYSize]
if opts.debug:
print("file %s: %s, dimensions: %s, %s, type: %s" % (myI, myF, DimensionsCheck[0], DimensionsCheck[1], myDataType[-1]))
# process allBands option
allBandsIndex = None
allBandsCount = 1
if opts.allBands:
try:
allBandsIndex = myAlphaList.index(opts.allBands)
except ValueError:
raise Exception("Error! allBands option was given but Band %s not found. Cannot proceed" % (opts.allBands))
allBandsCount = myFiles[allBandsIndex].RasterCount
if allBandsCount <= 1:
allBandsIndex = None
################################################################
# set up output file
################################################################
# open output file exists
if os.path.isfile(opts.outF) and not opts.overwrite:
if allBandsIndex is not None:
raise Exception("Error! allBands option was given but Output file exists, must use --overwrite option!")
if opts.debug:
print("Output file %s exists - filling in results into file" % (opts.outF))
myOut = gdal.Open(opts.outF, gdal.GA_Update)
if [myOut.RasterXSize, myOut.RasterYSize] != DimensionsCheck:
raise Exception("Error! Output exists, but is the wrong size. Use the --overwrite option to automatically overwrite the existing file")
myOutB = myOut.GetRasterBand(1)
myOutNDV = myOutB.GetNoDataValue()
myOutType = gdal.GetDataTypeName(myOutB.DataType)
else:
# remove existing file and regenerate
if os.path.isfile(opts.outF):
os.remove(opts.outF)
# create a new file
if opts.debug:
print("Generating output file %s" % (opts.outF))
# find data type to use
if not opts.type:
# use the largest type of the input files
myOutType = gdal.GetDataTypeName(max(myDataTypeNum))
else:
myOutType = opts.type
# create file
myOutDrv = gdal.GetDriverByName(opts.format)
myOut = myOutDrv.Create(
opts.outF, DimensionsCheck[0], DimensionsCheck[1], allBandsCount,
gdal.GetDataTypeByName(myOutType), opts.creation_options)
# set output geo info based on first input layer
myOut.SetGeoTransform(myFiles[0].GetGeoTransform())
myOut.SetProjection(myFiles[0].GetProjection())
if opts.NoDataValue is not None:
myOutNDV = opts.NoDataValue
else:
myOutNDV = DefaultNDVLookup[myOutType]
for i in range(1, allBandsCount + 1):
myOutB = myOut.GetRasterBand(i)
myOutB.SetNoDataValue(myOutNDV)
# write to band
myOutB = None
if opts.debug:
print("output file: %s, dimensions: %s, %s, type: %s" % (opts.outF, myOut.RasterXSize, myOut.RasterYSize, myOutType))
################################################################
# find block size to chop grids into bite-sized chunks
################################################################
# use the block size of the first layer to read efficiently
myBlockSize = myFiles[0].GetRasterBand(myBands[0]).GetBlockSize()
# find total x and y blocks to be read
nXBlocks = (int)((DimensionsCheck[0] + myBlockSize[0] - 1) / myBlockSize[0])
nYBlocks = (int)((DimensionsCheck[1] + myBlockSize[1] - 1) / myBlockSize[1])
myBufSize = myBlockSize[0] * myBlockSize[1]
if opts.debug:
print("using blocksize %s x %s" % (myBlockSize[0], myBlockSize[1]))
# variables for displaying progress
ProgressCt = -1
ProgressMk = -1
ProgressEnd = nXBlocks * nYBlocks * allBandsCount
################################################################
# start looping through each band in allBandsCount
################################################################
for bandNo in range(1, allBandsCount + 1):
################################################################
# start looping through blocks of data
################################################################
# store these numbers in variables that may change later
nXValid = myBlockSize[0]
nYValid = myBlockSize[1]
# loop through X-lines
for X in range(0, nXBlocks):
# in case the blocks don't fit perfectly
# change the block size of the final piece
if X == nXBlocks - 1:
nXValid = DimensionsCheck[0] - X * myBlockSize[0]
# find X offset
myX = X * myBlockSize[0]
# reset buffer size for start of Y loop
nYValid = myBlockSize[1]
myBufSize = nXValid * nYValid
# loop through Y lines
for Y in range(0, nYBlocks):
ProgressCt += 1
if 10 * ProgressCt / ProgressEnd % 10 != ProgressMk and not opts.quiet:
ProgressMk = 10 * ProgressCt / ProgressEnd % 10
from sys import version_info
if version_info >= (3, 0, 0):
exec('print("%d.." % (10*ProgressMk), end=" ")')
else:
exec('print 10*ProgressMk, "..",')
# change the block size of the final piece
if Y == nYBlocks - 1:
nYValid = DimensionsCheck[1] - Y * myBlockSize[1]
myBufSize = nXValid * nYValid
# find Y offset
myY = Y * myBlockSize[1]
# create empty buffer to mark where nodata occurs
myNDVs = None
# make local namespace for calculation
local_namespace = {}
# fetch data for each input layer
for i, Alpha in enumerate(myAlphaList):
# populate lettered arrays with values
if allBandsIndex is not None and allBandsIndex == i:
myBandNo = bandNo
else:
myBandNo = myBands[i]
myval = gdalnumeric.BandReadAsArray(myFiles[i].GetRasterBand(myBandNo),
xoff=myX, yoff=myY,
win_xsize=nXValid, win_ysize=nYValid)
# fill in nodata values
if myNDV[i] is not None:
if myNDVs is None:
myNDVs = numpy.zeros(myBufSize)
myNDVs.shape = (nYValid, nXValid)
myNDVs = 1 * numpy.logical_or(myNDVs == 1, myval == myNDV[i])
# add an array of values for this block to the eval namespace
local_namespace[Alpha] = myval
myval = None
# try the calculation on the array blocks
try:
myResult = eval(opts.calc, global_namespace, local_namespace)
except:
print("evaluation of calculation %s failed" % (opts.calc))
raise
# Propagate nodata values (set nodata cells to zero
# then add nodata value to these cells).
if myNDVs is not None:
myResult = ((1 * (myNDVs == 0)) * myResult) + (myOutNDV * myNDVs)
elif not isinstance(myResult, numpy.ndarray):
myResult = numpy.ones((nYValid, nXValid)) * myResult
# write data block to the output file
myOutB = myOut.GetRasterBand(bandNo)
gdalnumeric.BandWriteArray(myOutB, myResult, xoff=myX, yoff=myY)
if not opts.quiet:
print("100 - Done")
def files_mean(file_array, filename_out):
myFiles=[]
myNDV=[]
DimensionsCheck=None
allBandsCount=1
myOutNDV=-32767
for (i, filename) in enumerate(file_array):
if os.path.isfile(filename):
myFile = gdal.Open(filename, gdal.GA_ReadOnly)
if not myFile:
raise IOError("No such file or directory: '%s'" % myFile)
myFiles.append(myFile)
myNDV.append(myFile.GetRasterBand(1).GetNoDataValue())
if DimensionsCheck:
if DimensionsCheck != [myFile.RasterXSize, myFile.RasterYSize]:
raise Exception("Error! Dimensions of file %s (%i, %i) are different from other files (%i, %i). Cannot proceed" % (myF, myFile.RasterXSize, myFile.RasterYSize, DimensionsCheck[0], DimensionsCheck[1]))
else:
DimensionsCheck = [myFile.RasterXSize, myFile.RasterYSize]
if os.path.isfile(filename_out):
os.remove(filename_out)
myBlockSize=myFiles[0].GetRasterBand(1).GetBlockSize();
# store these numbers in variables that may change later
nXValid = myBlockSize[0]
nYValid = myBlockSize[1]
# find total x and y blocks to be read
nXBlocks = (int)((DimensionsCheck[0] + myBlockSize[0] - 1) / myBlockSize[0]);
nYBlocks = (int)((DimensionsCheck[1] + myBlockSize[1] - 1) / myBlockSize[1]);
myBufSize = myBlockSize[0]*myBlockSize[1]
myOutDrv = gdal.GetDriverByName('GTiff')
myOut = myOutDrv.Create(filename_out, DimensionsCheck[0], DimensionsCheck[1], 1,gdal.GDT_Float32)
# set output geo info based on first input layer
myOut.SetGeoTransform(myFiles[0].GetGeoTransform())
myOut.SetProjection(myFiles[0].GetProjection())
for bandNo in range(1,allBandsCount+1):
for X in range(0,nXBlocks):
if X==nXBlocks-1:
nXValid = DimensionsCheck[0] - X * myBlockSize[0]
myBufSize = nXValid*nYValid
# find X offset
myX=X*myBlockSize[0]
# reset buffer size for start of Y loop
nYValid = myBlockSize[1]
myBufSize = nXValid*nYValid
# loop through Y lines
for Y in range(0,nYBlocks):
# change the block size of the final piece
if Y==nYBlocks-1:
nYValid = DimensionsCheck[1] - Y * myBlockSize[1]
myBufSize = nXValid*nYValid
# find Y offset
myY=Y*myBlockSize[1]
# create empty buffer to mark where nodata occurs
myNDVs = None
# make local namespace for calculation
local_namespace = []
for (i, myFile) in enumerate(myFiles):
myval=gdalnumeric.BandReadAsArray(myFile.GetRasterBand(1),
xoff=myX, yoff=myY,
win_xsize=nXValid, win_ysize=nYValid)
# fill in nodata values
if myNDV[i] is not None:
if myNDVs is None:
myNDVs = numpy.zeros(myBufSize)
myNDVs.shape=(nYValid,nXValid)
myNDVs=1*numpy.logical_or(myNDVs==1, myval==myNDV[i])
local_namespace.append(myval)
myval=None
# calculate mean between bands using Numpy
myResult = numpy.mean(local_namespace, axis=0)
if myNDVs is not None:
myResult = ((1*(myNDVs==0))*myResult) + (myOutNDV*myNDVs)
elif not isinstance(myResult, numpy.ndarray):
myResult = numpy.ones( (nYValid,nXValid) ) * myResult
myOutB=myOut.GetRasterBand(bandNo)
gdalnumeric.BandWriteArray(myOutB, myResult, xoff=myX, yoff=myY)
return
# how to use it
files_mean(['./input_01.tif', './input_02.tif', './input_02.tif', ... './input_n.tif',],'./output.tif')
def main():
# initiate the parser
parser = argparse.ArgumentParser(
description='Summarize a set of rasters layers.')
parser.add_argument('files', nargs='+', help='input raster file(s)')
parser.add_argument('--outfile', '-o', required=True, help='output raster')
band_help = 'bands to summarize. ' + \
'single file: bands in this file to summarize (default all bands); ' + \
'multiple files: bands in corresponding files to summarize (default = 1)'
parser.add_argument('--bands', '-b', nargs='+', type=int, help=band_help)
function_help = 'function to apply to cell values across layers.' + \
'meannz gives a mean of the non-zero values.' + \
'count counts the number layers with non-negative values for each cell.' + \
'richness counts the number of layers with positive values for each cell.'
parser.add_argument('--function',
'-f',
dest='summary_function',
default='mean',
choices=[
'mean', 'median', 'max', 'sum', 'meannz', 'count',
'richness'
],
help="summarization function (default = 'mean')")
parser.add_argument(
'--block_size',
'-s',
nargs=2,
type=int,
help='x and y dimensions of blocks to process (default based on input)'
)
parser.add_argument(
'--nrows',
'-n',
type=int,
help=
'number of rows to process in a single block (block_size ignored if provided)'
)
parser.add_argument('--overwrite',
'-w',
action="store_true",
help='overwrite existing file')
parser.add_argument(
'--creation-option',
'--co',
dest='creation_options',
default=[],
action='append',
help=
'passes one or more creation options to the output format driver multiple'
)
parser.add_argument('--quiet',
'-q',
action="store_true",
help='supress messages')
# read arguments from the command line
args = parser.parse_args()
# summarize multiple bands of same file or bands across files
if len(args.files) == 1:
# use all bands if not specified
if not args.bands:
b = gdal.Open(args.files[0], gdal.GA_ReadOnly).RasterCount
args.bands = [i for i in range(1, b + 1)]
# elif len(args.bands) == 1:
# raise Exception('For a single input file, provide multiple bands to summarize over.')
args.files = [args.files[0]] * len(args.bands)
else:
# use band 1 if not specified
if not args.bands:
args.bands = [1] * len(args.files)
elif len(args.bands) == 1:
args.bands = [args.bands[0]] * len(args.files)
elif len(args.bands) != len(args.files):
raise Exception(
'The number of bands must be 1 or equal to the number of input files.'
)
# set up some default nodatavalues for each datatype
ndv_lookup = {
'Byte': 255,
'UInt16': 65535,
'Int16': -32767,
'UInt32': 4294967293,
'Int32': -2147483647,
'Float32': 3.402823466E+38,
'Float64': 1.7976931348623158E+308
}
stack = []
in_type = []
in_ndv = []
raster_dim = None
block_size = None
out_transform = None
out_projection = None
# pass over files to get metadata and check arguments
for i in range(len(args.files)):
f = args.files[i]
b = args.bands[i]
s = gdal.Open(f, gdal.GA_ReadOnly)
# check that file exists
if not s:
raise IOError('No such file or directory: {}'.format(f))
# check that band is valid
if b > s.RasterCount:
raise IOError('Invalid band number ({}) for file: {}'.format(b, f))
# get specified band
r = s.GetRasterBand(b)
# store raster file
stack.append(s)
# metadata
in_type.append(r.DataType)
dt_name = gdal.GetDataTypeName(r.DataType).lower()
in_ndv.append(np.float64(r.GetNoDataValue()).astype(dt_name))
# check that the dimensions of each layer are the same
if raster_dim:
if raster_dim != [s.RasterXSize, s.RasterYSize]:
raise Exception('Input files have different dimensions.')
else:
raster_dim = [s.RasterXSize, s.RasterYSize]
# block size to chop grids into bite-sized chunks
if not block_size:
# use the block size of the first layer to read efficiently
# or use user provided block size
if not args.nrows:
if not args.block_size:
block_size = s.GetRasterBand(1).GetBlockSize()
else:
# block size can't be larger than raster dimensions
block_size = np.minimum(args.block_size,
raster_dim).tolist()
else:
block_size = [raster_dim[0], min(raster_dim[1], args.nrows)]
# get geo info from first layer
if not out_transform:
out_transform = s.GetGeoTransform()
if not out_projection:
out_projection = s.GetProjection()
# prepare output file
if os.path.isfile(args.outfile) and not args.overwrite:
raise Exception(
"Output exists, use the --overwrite to overwrite the existing file"
)
else:
# remove existing file and regenerate
if os.path.isfile(args.outfile):
os.remove(args.outfile)
# for sum use the largest type of the input files
# otherise use a value suitable for the function
if args.summary_function == 'sum':
out_type = gdal.GetDataTypeName(max(in_type))
out_type_np = out_type.lower()
np_nan = 0
elif args.summary_function == 'mean':
out_type = 'Float32'
out_type_np = out_type.lower()
np_nan = np.nan
elif args.summary_function == 'median':
out_type = 'Float32'
out_type_np = out_type.lower()
np_nan = np.nan
elif args.summary_function == 'max':
out_type = 'Float32'
out_type_np = out_type.lower()
np_nan = np.nan
elif args.summary_function == 'meannz':
out_type = 'Float32'
out_type_np = out_type.lower()
np_nan = np.nan
elif args.summary_function == 'count':
out_type = 'Int16'
out_type_np = out_type.lower()
np_nan = -1
elif args.summary_function == 'richness':
out_type = 'Int16'
out_type_np = out_type.lower()
np_nan = -1
# create file
out_driver = gdal.GetDriverByName('GTiff')
r_out = out_driver.Create(args.outfile, raster_dim[0], raster_dim[1],
1, gdal.GetDataTypeByName(out_type),
args.creation_options)
# set output geo info based on first input layer
r_out.SetGeoTransform(out_transform)
r_out.SetProjection(out_projection)
# set no data value
out_ndv = ndv_lookup[out_type]
out_ndv_np = np.float64(out_ndv).astype(out_type_np)
r_out.GetRasterBand(1).SetNoDataValue(out_ndv)
# find total x and y blocks to be read
xblocks = math.ceil(raster_dim[0] / block_size[0])
yblocks = math.ceil(raster_dim[1] / block_size[1])
# loop through blocks of data
# store these numbers in variables that may change later
xvalid = block_size[0]
yvalid = block_size[1]
# variables for displaying progress
progress_cnt = -1
progress_stp = 0
progress_end = xblocks * yblocks
progress_stps = [round(progress_end * i / 100) for i in range(0, 101, 10)]
# message
if not args.quiet:
m = "Processing {} X {} raster (cols X rows) in {} blocks of {} X {}"
m = m.format(raster_dim[0], raster_dim[1], xblocks * yblocks,
block_size[0], block_size[1])
print(m)
# loop through x dimension
for x in range(0, xblocks):
# in case the blocks don't fit perfectly
# change the block size of the final piece
if x == xblocks - 1:
xvalid = raster_dim[0] - x * block_size[0]
# find x offset
x_off = x * block_size[0]
# reset buffer size for start of Y loop
yvalid = block_size[1]
# loop through y dimension
for y in range(0, yblocks):
# progress bar
progress_cnt += 1
if progress_cnt == progress_stps[progress_stp] and not args.quiet:
print('%d...' % (10 * progress_stp), end="", flush=True)
progress_stp += 1
# change the block size of the final piece
if y == yblocks - 1:
yvalid = raster_dim[1] - y * block_size[1]
# find y offset
y_off = y * block_size[1]
# create empty buffer to mark where nodata occurs
ndv_buffer = None
# make array to store block
block = np.empty(shape=(len(stack), yvalid, xvalid),
dtype='float32')
# fetch data for each input layer
for i in range(len(stack)):
vals = gdalnumeric.BandReadAsArray(stack[i].GetRasterBand(
args.bands[i]),
xoff=x_off,
yoff=y_off,
win_xsize=xvalid,
win_ysize=yvalid)
if out_type != "Int16":
vals = vals.astype(out_type_np)
# fill in nodata values
if in_ndv[i] is not None:
vals[vals == in_ndv[i]] = np_nan
# add block to array
block[i] = vals
vals = None
# ignore empty slice warnings
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=RuntimeWarning)
if args.summary_function == 'sum':
result = np.nansum(block, axis=0)
elif args.summary_function == 'mean':
result = np.nanmean(block, axis=0)
elif args.summary_function == 'median':
result = np.nanmedian(block, axis=0)
elif args.summary_function == 'max':
result = np.nanmax(block, axis=0)
elif args.summary_function == 'meannz':
block = np.ma.masked_equal(block, 0)
result = np.nanmean(block, axis=0)
elif args.summary_function == 'count':
result = np.nansum(block >= 0, axis=0)
elif args.summary_function == 'richness':
result = np.nansum(block > 0, axis=0)
# replace nan with no data value
result[np.isnan(result)] = out_ndv_np
# write data block to the output file
gdalnumeric.BandWriteArray(r_out.GetRasterBand(1),
result,
xoff=x_off,
yoff=y_off)
# end progress bar
if not args.quiet:
print('100 - Done')
def doit(opts, args):
# pylint: disable=unused-argument
if opts.debug:
print("gdal_calc.py starting calculation %s" % (opts.calc))
# set up global namespace for eval with all functions of gdalnumeric
global_namespace = dict([(key, getattr(gdalnumeric, key))
for key in dir(gdalnumeric) if not key.startswith('__')])
if opts.user_namespace:
global_namespace.update(opts.user_namespace)
if not opts.calc:
raise Exception("No calculation provided.")
elif not opts.outF and opts.format.upper() != 'MEM':
raise Exception("No output file provided.")
if opts.format is None:
opts.format = GetOutputDriverFor(opts.outF)
if not hasattr(opts, "color_table"):
opts.color_table = None
if not opts.extent:
opts.extent = EXTENT_IGNORE
else:
opts.extent = parse_extent(opts.extent)
compatible_gt_eps = 0.000001
gt_diff_support = {
GT_INCOMPATIBLE_OFFSET: opts.extent != EXTENT_FAIL,
GT_INCOMPATIBLE_PIXEL_SIZE: False,
GT_INCOMPATIBLE_ROTATION: False,
GT_NON_ZERO_ROTATION: False,
}
gt_diff_error = {
GT_INCOMPATIBLE_OFFSET: 'different offset',
GT_INCOMPATIBLE_PIXEL_SIZE: 'different pixel size',
GT_INCOMPATIBLE_ROTATION: 'different rotation',
GT_NON_ZERO_ROTATION: 'non zero rotation',
}
################################################################
# fetch details of input layers
################################################################
# set up some lists to store data for each band
myFileNames = [] # input filenames
myFiles = [] # input DataSets
myBands = [] # input bands
myAlphaList = [] # input alpha letter that represents each input file
myDataType = [] # string representation of the datatype of each input file
myDataTypeNum = [] # datatype of each input file
myNDV = [] # nodatavalue for each input file
DimensionsCheck = None # dimensions of the output
Dimensions = [] # Dimensions of input files
ProjectionCheck = None # projection of the output
GeoTransformCheck = None # GeoTransform of the output
GeoTransforms = [] # GeoTransform of each input file
GeoTransformDiffer = False # True if we have inputs with different GeoTransforms
myTempFileNames = [] # vrt filename from each input file
myAlphaFileLists = [] # list of the Alphas which holds a list of inputs
# loop through input files - checking dimensions
for alphas, filenames in opts.input_files.items():
if isinstance(filenames, (list, tuple)):
# alpha is a list of files
myAlphaFileLists.append(alphas)
elif is_path_like(filenames) or isinstance(filenames, gdal.Dataset):
# alpha is a single filename or a Dataset
filenames = [filenames]
alphas = [alphas]
else:
# I guess this alphas should be in the global_namespace,
# It would have been better to pass it as user_namepsace, but I'll accept it anyway
global_namespace[alphas] = filenames
continue
for alpha, filename in zip(alphas*len(filenames), filenames):
if not alpha.endswith("_band"):
# check if we have asked for a specific band...
if "%s_band" % alpha in opts.input_files:
myBand = opts.input_files["%s_band" % alpha]
else:
myBand = 1
myF_is_ds = not is_path_like(filename)
if myF_is_ds:
myFile = filename
filename = None
else:
filename = str(filename)
myFile = gdal.Open(filename, gdal.GA_ReadOnly)
if not myFile:
raise IOError("No such file or directory: '%s'" % filename)
myFileNames.append(filename)
myFiles.append(myFile)
myBands.append(myBand)
myAlphaList.append(alpha)
dt = myFile.GetRasterBand(myBand).DataType
myDataType.append(gdal.GetDataTypeName(dt))
myDataTypeNum.append(dt)
myNDV.append(None if opts.hideNoData else myFile.GetRasterBand(myBand).GetNoDataValue())
# check that the dimensions of each layer are the same
myFileDimensions = [myFile.RasterXSize, myFile.RasterYSize]
if DimensionsCheck:
if DimensionsCheck != myFileDimensions:
GeoTransformDiffer = True
if opts.extent in [EXTENT_IGNORE, EXTENT_FAIL]:
raise Exception("Error! Dimensions of file %s (%i, %i) are different from other files (%i, %i). Cannot proceed" %
(filename, myFileDimensions[0], myFileDimensions[1], DimensionsCheck[0], DimensionsCheck[1]))
else:
DimensionsCheck = myFileDimensions
# check that the Projection of each layer are the same
myProjection = myFile.GetProjection()
if ProjectionCheck:
if opts.projectionCheck and ProjectionCheck != myProjection:
raise Exception(
"Error! Projection of file %s %s are different from other files %s. Cannot proceed" %
(filename, myProjection, ProjectionCheck))
else:
ProjectionCheck = myProjection
# check that the GeoTransforms of each layer are the same
myFileGeoTransform = myFile.GetGeoTransform(can_return_null=True)
if opts.extent == EXTENT_IGNORE:
GeoTransformCheck = myFileGeoTransform
else:
Dimensions.append(myFileDimensions)
GeoTransforms.append(myFileGeoTransform)
if not GeoTransformCheck:
GeoTransformCheck = myFileGeoTransform
else:
my_gt_diff = gt_diff(GeoTransformCheck, myFileGeoTransform, eps=compatible_gt_eps, diff_support=gt_diff_support)
if my_gt_diff not in [GT_SAME, GT_ALMOST_SAME]:
GeoTransformDiffer = True
if my_gt_diff not in [GT_COMPATIBLE_DIFF]:
raise Exception(
"Error! GeoTransform of file {} {} is incompatible ({}), first file GeoTransform is {}. Cannot proceed".
format(filename, myFileGeoTransform, gt_diff_error[my_gt_diff], GeoTransformCheck))
if opts.debug:
print("file %s: %s, dimensions: %s, %s, type: %s" % (
alpha, filename, DimensionsCheck[0], DimensionsCheck[1], myDataType[-1]))
# process allBands option
allBandsIndex = None
allBandsCount = 1
if opts.allBands:
if len(opts.calc) > 1:
raise Exception("Error! --allBands implies a single --calc")
try:
allBandsIndex = myAlphaList.index(opts.allBands)
except ValueError:
raise Exception("Error! allBands option was given but Band %s not found. Cannot proceed" % (opts.allBands))
allBandsCount = myFiles[allBandsIndex].RasterCount
if allBandsCount <= 1:
allBandsIndex = None
else:
allBandsCount = len(opts.calc)
if opts.extent not in [EXTENT_IGNORE, EXTENT_FAIL] and (GeoTransformDiffer or not isinstance(opts.extent, EXTENT)):
raise Exception('Error! mixing different GeoTransforms/Extents is not supported yet.')
################################################################
# set up output file
################################################################
# open output file exists
if opts.outF and os.path.isfile(opts.outF) and not opts.overwrite:
if allBandsIndex is not None:
raise Exception("Error! allBands option was given but Output file exists, must use --overwrite option!")
if len(opts.calc) > 1:
raise Exception("Error! multiple calc options were given but Output file exists, must use --overwrite option!")
if opts.debug:
print("Output file %s exists - filling in results into file" % (opts.outF))
myOut = gdal.Open(opts.outF, gdal.GA_Update)
if myOut is None:
error = 'but cannot be opened for update'
elif [myOut.RasterXSize, myOut.RasterYSize] != DimensionsCheck:
error = 'but is the wrong size'
elif ProjectionCheck and ProjectionCheck != myOut.GetProjection():
error = 'but is the wrong projection'
elif GeoTransformCheck and GeoTransformCheck != myOut.GetGeoTransform(can_return_null=True):
error = 'but is the wrong geotransform'
else:
error = None
if error:
raise Exception("Error! Output exists, %s. Use the --overwrite option to automatically overwrite the existing file" % error)
myOutB = myOut.GetRasterBand(1)
myOutNDV = myOutB.GetNoDataValue()
myOutType = myOutB.DataType
else:
if opts.outF:
# remove existing file and regenerate
if os.path.isfile(opts.outF):
os.remove(opts.outF)
# create a new file
if opts.debug:
print("Generating output file %s" % (opts.outF))
else:
opts.outF = ''
# find data type to use
if not opts.type:
# use the largest type of the input files
myOutType = max(myDataTypeNum)
else:
myOutType = opts.type
if isinstance(myOutType, str):
myOutType = gdal.GetDataTypeByName(myOutType)
# create file
myOutDrv = gdal.GetDriverByName(opts.format)
myOut = myOutDrv.Create(
opts.outF, DimensionsCheck[0], DimensionsCheck[1], allBandsCount,
myOutType, opts.creation_options)
# set output geo info based on first input layer
if not GeoTransformCheck:
GeoTransformCheck = myFiles[0].GetGeoTransform(can_return_null=True)
if GeoTransformCheck:
myOut.SetGeoTransform(GeoTransformCheck)
if not ProjectionCheck:
ProjectionCheck = myFiles[0].GetProjection()
if ProjectionCheck:
myOut.SetProjection(ProjectionCheck)
if opts.NoDataValue is None:
myOutNDV = None if opts.hideNoData else DefaultNDVLookup[myOutType] # use the default noDataValue for this datatype
elif isinstance(opts.NoDataValue, str) and opts.NoDataValue.lower() == 'none':
myOutNDV = None # not to set any noDataValue
else:
myOutNDV = opts.NoDataValue # use the given noDataValue
for i in range(1, allBandsCount + 1):
myOutB = myOut.GetRasterBand(i)
if myOutNDV is not None:
myOutB.SetNoDataValue(myOutNDV)
if opts.color_table:
# set color table and color interpretation
if is_path_like(opts.color_table):
raise Exception('Error! reading a color table from a file is not supported yet')
myOutB.SetRasterColorTable(opts.color_table)
myOutB.SetRasterColorInterpretation(gdal.GCI_PaletteIndex)
myOutB = None # write to band
myOutTypeName = gdal.GetDataTypeName(myOutType)
if opts.debug:
print("output file: %s, dimensions: %s, %s, type: %s" % (opts.outF, myOut.RasterXSize, myOut.RasterYSize, myOutTypeName))
################################################################
# find block size to chop grids into bite-sized chunks
################################################################
# use the block size of the first layer to read efficiently
myBlockSize = myFiles[0].GetRasterBand(myBands[0]).GetBlockSize()
# find total x and y blocks to be read
nXBlocks = (int)((DimensionsCheck[0] + myBlockSize[0] - 1) / myBlockSize[0])
nYBlocks = (int)((DimensionsCheck[1] + myBlockSize[1] - 1) / myBlockSize[1])
myBufSize = myBlockSize[0] * myBlockSize[1]
if opts.debug:
print("using blocksize %s x %s" % (myBlockSize[0], myBlockSize[1]))
# variables for displaying progress
ProgressCt = -1
ProgressMk = -1
ProgressEnd = nXBlocks * nYBlocks * allBandsCount
################################################################
# start looping through each band in allBandsCount
################################################################
for bandNo in range(1, allBandsCount + 1):
################################################################
# start looping through blocks of data
################################################################
# store these numbers in variables that may change later
nXValid = myBlockSize[0]
nYValid = myBlockSize[1]
# loop through X-lines
for X in range(0, nXBlocks):
# in case the blocks don't fit perfectly
# change the block size of the final piece
if X == nXBlocks - 1:
nXValid = DimensionsCheck[0] - X * myBlockSize[0]
# find X offset
myX = X * myBlockSize[0]
# reset buffer size for start of Y loop
nYValid = myBlockSize[1]
myBufSize = nXValid * nYValid
# loop through Y lines
for Y in range(0, nYBlocks):
ProgressCt += 1
if 10 * ProgressCt / ProgressEnd % 10 != ProgressMk and not opts.quiet:
ProgressMk = 10 * ProgressCt / ProgressEnd % 10
from sys import version_info
if version_info >= (3, 0, 0):
exec('print("%d.." % (10*ProgressMk), end=" ")')
else:
exec('print 10*ProgressMk, "..",')
# change the block size of the final piece
if Y == nYBlocks - 1:
nYValid = DimensionsCheck[1] - Y * myBlockSize[1]
myBufSize = nXValid * nYValid
# find Y offset
myY = Y * myBlockSize[1]
# create empty buffer to mark where nodata occurs
myNDVs = None
# make local namespace for calculation
local_namespace = {}
val_lists = defaultdict(list)
# fetch data for each input layer
for i, Alpha in enumerate(myAlphaList):
# populate lettered arrays with values
if allBandsIndex is not None and allBandsIndex == i:
myBandNo = bandNo
else:
myBandNo = myBands[i]
myval = gdalnumeric.BandReadAsArray(myFiles[i].GetRasterBand(myBandNo),
xoff=myX, yoff=myY,
win_xsize=nXValid, win_ysize=nYValid)
if myval is None:
raise Exception('Input block reading failed from filename %s' % filename[i])
# fill in nodata values
if myNDV[i] is not None:
# myNDVs is a boolean buffer.
# a cell equals to 1 if there is NDV in any of the corresponding cells in input raster bands.
if myNDVs is None:
# this is the first band that has NDV set. we initializes myNDVs to a zero buffer
# as we didn't see any NDV value yet.
myNDVs = numpy.zeros(myBufSize)
myNDVs.shape = (nYValid, nXValid)
myNDVs = 1 * numpy.logical_or(myNDVs == 1, myval == myNDV[i])
# add an array of values for this block to the eval namespace
if Alpha in myAlphaFileLists:
val_lists[Alpha].append(myval)
else:
local_namespace[Alpha] = myval
myval = None
for lst in myAlphaFileLists:
local_namespace[lst] = val_lists[lst]
# try the calculation on the array blocks
calc = opts.calc[bandNo-1 if len(opts.calc) > 1 else 0]
try:
myResult = eval(calc, global_namespace, local_namespace)
except:
print("evaluation of calculation %s failed" % (calc))
raise
# Propagate nodata values (set nodata cells to zero
# then add nodata value to these cells).
if myNDVs is not None and myOutNDV is not None:
myResult = ((1 * (myNDVs == 0)) * myResult) + (myOutNDV * myNDVs)
elif not isinstance(myResult, numpy.ndarray):
myResult = numpy.ones((nYValid, nXValid)) * myResult
# write data block to the output file
myOutB = myOut.GetRasterBand(bandNo)
if gdalnumeric.BandWriteArray(myOutB, myResult, xoff=myX, yoff=myY) != 0:
raise Exception('Block writing failed')
myOutB = None # write to band
# remove temp files
for idx, tempFile in enumerate(myTempFileNames):
myFiles[idx] = None
os.remove(tempFile)
gdal.ErrorReset()
myOut.FlushCache()
if gdal.GetLastErrorMsg() != '':
raise Exception('Dataset writing failed')
if not opts.quiet:
print("100 - Done")
return myOut
import pandas import numpy as np import matplotlib.pyplot as plt from osgeo import gdal, gdalnumeric #filename = "Users/mattolson/data/ASO/CASI/20170220/174008/CASI_2017_02_20_174008_atm_ort" filename = "/Users/mattolson/data/ASO/CASI/20170220/174008/CASI_2017_02_20_174008_atm_ort" casi = gdal.Open(filename) b1 = casi.GetRasterBand(1) b1arr = gdalnumeric.BandReadAsArray(b1) b1mask = np.ma.masked_values(b1arr, b1.GetNoDataValue(), copy=False) plt.figure() plt.imshow(b1mask) plt.legend() plt.show() # np.ma.vstack or dstack # multi-dimentional array