def create_tif_mask(in_fname, out_fname):
# Open the raster file
raster = gdal.Open(in_fname, gdal.GA_ReadOnly)
band_1 = raster.GetRasterBand(1)
# Read the data as a numpy array
data_1 = gdal_array.BandReadAsArray(band_1)
# Create a boolean band with all 1's
mask = (data_1 >= 0).astype(int)
assert np.mean(mask) == 1
# Write the output mask
driver = gdal.GetDriverByName("GTiff")
ds_out = driver.Create(out_fname, raster.RasterXSize, raster.RasterYSize,
1, band_1.DataType)
gdal_array.CopyDatasetInfo(raster, ds_out)
band_out = ds_out.GetRasterBand(1)
gdal_array.BandWriteArray(band_out, mask)
# Close the datasets
band_1 = None
raster = None
band_out = None
ds_out = None
def render_image(self, extent, size):
ds = gdal.Warp(
"",
self.parent.gdal_dataset(),
options=gdal.WarpOptions(
width=size[0],
height=size[1],
outputBounds=extent,
format="MEM",
warpOptions=['UNIFIED_SRC_NODATA=ON'],
dstAlpha=True,
),
)
result = PIL.Image.new("RGBA", size, (0, 0, 0, 0))
band_count = ds.RasterCount
array = numpy.zeros((size[1], size[0], band_count), numpy.uint8)
for i in range(band_count):
array[:, :,
i] = gdal_array.BandReadAsArray(ds.GetRasterBand(i + 1), )
ds = None
wnd = PIL.Image.fromarray(array)
result.paste(wnd)
return result
def img_to_arr(input_file, dim_ordering='channels_last', dtype='float32'):
"""Takes in a raster file as input and converts to numpy array"""
bands = [
input_file.GetRasterBand(i)
for i in range(1, input_file.RasterCount + 1)
]
arr = np.array([gdal_array.BandReadAsArray(band)
for band in bands]).astype(dtype)
if dim_ordering == 'channels_last':
arr = np.transpose(arr, [1, 2, 0])
return arr
def get_clip(self, bnds):
"""get the part of our grid clipped by bnds (from get_bounds())"""
try:
rasterDat = gdal_array.BandReadAsArray(self.img.GetRasterBand(1),
bnds.fcols, bnds.frows,
bnds.fcolcnt, bnds.frowcnt)
except:
print bnds.fcols, bnds.fcole, bnds.fcolcnt, bnds.frows, \
bnds.frowe, bnds.frowcnt
raise
return rasterDat
def scale_query_to_tile(self, dsquery, dstile, tx, ty, tz): """Scales down query dataset to the tile dataset""" querysize = dsquery.RasterXSize tilesize = dstile.RasterXSize # Scaling by PIL (Python Imaging Library) - improved Lanczos # mod XX: to accomodate with 2 band hilshades array = numpy.zeros((querysize, querysize, 4), numpy.uint8) white=numpy.zeros((querysize, querysize), numpy.uint8) white.fill(255) #for i in [0,1,2]: # array[:,:,i] = gdalarray.BandReadAsArray(dsquery.GetRasterBand(1), 0, 0, querysize, querysize) hillshade = 255 - gdalarray.BandReadAsArray(dsquery.GetRasterBand(1), 0, 0, querysize, querysize) mask = gdalarray.BandReadAsArray(dsquery.GetRasterBand(2), 0, 0, querysize, querysize) / 255 array[:,:,3] = hillshade*mask im = Image.fromarray(array, 'RGBA') # Always four bands im1 = im.filter(ImageFilter.BLUR).resize((tilesize,tilesize), Image.ANTIALIAS) #im1.save(tilefilename,self.tiledriver) buf= StringIO.StringIO() im1.save(buf,self.tiledriver) self.store.writeImage(tx, ty, tz, buf.getvalue())
def read_image(x1, y1, x2, y2, srs):
width = height = 500
ds_img = gdal.Warp('',
ds,
options=gdal.WarpOptions(width=width,
height=height,
outputBounds=(x1, y1, x2,
y2),
dstSRS=srs,
format='MEM'))
array = numpy.zeros((height, width, band_count), numpy.uint8)
for i in range(band_count):
band = ds_img.GetRasterBand(i + 1)
array[:, :, i] = gdal_array.BandReadAsArray(band)
img = Image.fromarray(array)
return img
def resample_antialias(path, dsquery, dstile):
querysize = dsquery.RasterXSize
tile_size = dstile.RasterXSize
array = numpy.zeros((querysize, querysize, 4), numpy.uint8)
for i in range(dstile.RasterCount):
array[:, :, i] = gdalarray.BandReadAsArray( # NOQA
dsquery.GetRasterBand(i + 1), 0, 0, querysize, querysize)
im = Image.fromarray(array, 'RGBA') # Always four bands
im1 = im.resize((tile_size, tile_size), Image.ANTIALIAS)
if path.exists():
im0 = Image.open(str(path))
im1 = Image.composite(im1, im0, im1)
ensure_dir_exists(path.parent)
im1.save(str(path), 'PNG')
def reclassifyRaster(raster, bins, newbinvalues):
#turn raster into numpy array
rasterIn = gdal.Open(raster)
arr = gdal_array.BandReadAsArray(rasterIn.GetRasterBand(1))
print arr
#conditional loop to reclassify
arr[(bins[5] >= arr) & (arr > bins[4])] = newbinvalues[4]
arr[(bins[4] >= arr) & (arr > bins[3])] = newbinvalues[3]
arr[(bins[3] >= arr) & (arr > bins[2])] = newbinvalues[2]
arr[(bins[2] >= arr) & (arr > bins[1])] = newbinvalues[1]
arr[(bins[1] >= arr) & (arr > bins[0])] = newbinvalues[0]
print arr
#Array to raster
rasterOut = raster.split('.')[0] + '_reclassified.TIF'
array2raster(arr, rasterIn, rasterOut)
return rasterOut
def single_band_to_indexed_numpy(band, nodata=-3.40282e+038, return_index=True):
""" Convert a dataset band into an indexed array
:param band: a gdal band
:param nodata: pixel no data value
:param return_index: a bool to return the array index
:return: (index, r_indexed) or r_indexed , depending on return_index
"""
try:
# convert a gdal band to a numpyarray
px = gdar.BandReadAsArray(band, 0, 0, band.XSize, band.YSize)
index = np.nonzero(px > nodata) # define the index
r_indexed = px[index] # apply the index to the band data
if return_index:
return index, r_indexed
else:
return r_indexed
except RuntimeError as err:
raise err
except Exception as e:
raise e
def apply_index_to_single_band(band, index, return_index=False):
""" Apply an existing index to a band
:param band: a gdal band
:param nodata: a numpy index
:param return_index: a bool to return the array index
:return: (index, r_indexed) or r_indexed , depending on return_index
"""
try:
# convert a gdal band to a numpyarray
px = gdar.BandReadAsArray(band, 0, 0, band.XSize, band.YSize)
r_indexed = px[index] # apply the index to the band data
if return_index:
return index, r_indexed
else:
return r_indexed
except RuntimeError as err:
raise err
except Exception as e:
raise e
def band_values(self, **kwargs):
"""
Method to read band from arguments or from initialized raster.
Will mask values defined in the band NoDataValue.
:param kwargs:
'band_number': band_number to read instead of the one given with
the initialize call.
:return: Numpy masked array
"""
band_number = kwargs.get('band_number', self.band_number)
band = self.file.GetRasterBand(band_number)
values = np.ma.masked_values(
gdal_array.BandReadAsArray(band),
band.GetNoDataValue(),
copy=False
)
del band
return values
def __init__(self, outdir, ref_img_fp, tgt_img_fp,
band_list=[1, 2, 3, 4],
band_names=['Blue', 'Green', 'Red', 'NIR'],
max_spectra = [3000, 3000, 3000, 8000],
loess_frac=0.15,
delete_working_files=True):
os.chdir(outdir)
self.get_overlap_areas(ref_img_fp, tgt_img_fp, outdir)
# Do LORACCS correction on target image
ref_img = gdal.Open('Ref_training_pixels.tif')
tgt_img = gdal.Open('Tgt_training_pixels.tif')
loraccs_img_fp = 'LORACCS_normalized_img.tif' # File name of transformed image
tgt_img_2 = rasterio.open(tgt_img_fp) # Will pull metadata from original image for LORACCS image
for num, band in enumerate(band_list):
# Get the band data as a numpy array
ref_data = ref_img.GetRasterBand(band)
tgt_data = tgt_img.GetRasterBand(band)
ref_img_band = gdal_array.BandReadAsArray(ref_data)
tgt_img_band = gdal_array.BandReadAsArray(tgt_data)
# Do LORACCS normalization
loraccs_array = self.run_loraccs(ref_img_band, tgt_img_band, band, band_names[num],
max_spectra[num], loess_frac, tgt_img_fp, outdir)
# Write transformed array to new image
with rasterio.Env():
profile = tgt_img_2.profile
profile.update(nodata=0)
if num == 0:
with rasterio.open(loraccs_img_fp, 'w', **profile) as dst:
dst.write(loraccs_array, band)
else:
with rasterio.open(loraccs_img_fp, 'r+', **profile) as dst:
dst.write(loraccs_array, band)
# Get the diagonal pixels from the LORACCS image to run quality assessment
self.get_qa_pixels('overlap.shp', loraccs_img_fp, outdir)
# Generate NRMSE file
self.nrmse = self.get_NRMSE(band_list, band_names, outdir)
print(self.nrmse)
# Print completed message
print('LORACCS transformation complete. New image file can be found in the given directory.')
print('File name: LORACCS_normalized_img.tif')
# Delete working files
if delete_working_files == True:
print('Removing working files.')
working_files = ['Ref_training_pixels.tif', 'QA_diag_lines.shp',
'Reference_clip_overlap.tif', 'overlap.prj', 'QA_diag_lines.cpg',
'LORACCS_image_overlap_clip.tif', 'QA_diag_lines.prj',
'overlap.shp', 'QA_diag_lines.dbf', 'Ref_assessment_pixels.tif',
'overlap.cpg', 'Tgt_assessment_pixels.tif', 'LORACCS_assessment_pixels.tif',
'Target_clip_overlap.tif', 'Tgt_training_pixels.tif', 'QA_diag_lines.shx',
'overlap.shx', 'overlap.dbf']
for name in band_names:
band_files = ['%s_2d_hist.png' %name,
'%s_df.csv' %name,
'%s full spectra.csv' %name]
working_files.extend(band_files[:])
for file in working_files:
os.remove(file)
def renderTile(self, tile):
import PIL.Image as PILImage
import StringIO
bounds = tile.bounds()
im = None
# If the image is entirely outside the bounds, don't bother doing anything with it:
# just return an 'empty' tile.
if not (bounds[2] < self.data_extent[0]
or bounds[0] > self.data_extent[2]
or bounds[3] < self.data_extent[1]
or bounds[1] > self.data_extent[3]):
tile_offset_left = tile_offset_top = 0
target_size = tile.size()
off_x = int(
(bounds[0] - self.geo_transform[0]) / self.geo_transform[1])
off_y = int(
(bounds[3] - self.geo_transform[3]) / self.geo_transform[5])
width_x = int((
(bounds[2] - self.geo_transform[0]) / self.geo_transform[1]) -
off_x)
width_y = int((
(bounds[1] - self.geo_transform[3]) / self.geo_transform[5]) -
off_y)
# Prevent from reading off the sides of an image
if off_x + width_x > self.ds.RasterXSize:
oversize_right = off_x + width_x - self.ds.RasterXSize
target_size = [
target_size[0] -
int(float(oversize_right) / width_x * target_size[0]),
target_size[1]
]
width_x = self.ds.RasterXSize - off_x
if off_x < 0:
oversize_left = -off_x
tile_offset_left = int(
float(oversize_left) / width_x * target_size[0])
target_size = [
target_size[0] -
int(float(oversize_left) / width_x * target_size[0]),
target_size[1],
]
width_x = width_x + off_x
off_x = 0
if off_y + width_y > self.ds.RasterYSize:
oversize_bottom = off_y + width_y - self.ds.RasterYSize
target_size = [
target_size[0], target_size[1] -
round(float(oversize_bottom) / width_y * target_size[1])
]
width_y = self.ds.RasterYSize - off_y
if off_y < 0:
oversize_top = -off_y
tile_offset_top = int(
float(oversize_top) / width_y * target_size[1])
target_size = [
target_size[0],
target_size[1] -
int(float(oversize_top) / width_y * target_size[1]),
]
width_y = width_y + off_y
off_y = 0
bands = self.ds.RasterCount
array = numpy.zeros((target_size[1], target_size[0], bands),
numpy.uint8)
for i in range(bands):
array[:, :, i] = gdalarray.BandReadAsArray(
self.ds.GetRasterBand(i + 1), off_x, off_y, width_x,
width_y, target_size[0], target_size[1])
im = PIL.Image.fromarray(array)
big = PIL.Image.new("RGBA", tile.size(), (0, 0, 0, 0))
if im:
big.paste(im, (tile_offset_left, tile_offset_top))
buffer = StringIO.StringIO()
big.save(buffer, self.extension)
buffer.seek(0)
tile.data = buffer.read()
return tile.data
def makelines(path,algo="HoughLinesP", param={} ):
d = gdal.Open(path)
band = d.GetRasterBand(1)
image = gdar.BandReadAsArray(band)
d=None
# set to uint8
imagebw = image.astype(np.uint8, copy=False)
#define the output image
cdst = cv2.cvtColor(imagebw, cv2.COLOR_GRAY2BGR)
# image = cv2.imread(path)
# image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# image = cv2.GaussianBlur(image, (5, 5), 0)
# cv2.imshow("Blurred", image)
# canny = cv2.Canny(image, 30, 150)
# cv2.imshow("Canny", canny)
# cv2.waitKey(0)
if (algo=="HoughLinesP"): # HoughLinesP
if not param:
lines = cv2.HoughLinesP(imagebw, 1, math.pi / 180.0, 40, np.array([]), 50, 10)
else:
lines = cv2.HoughLinesP(imagebw, param['rho'], param['theta'], param['threshold'],
np.array([]), param['minlinelenght'], param['maxlinegap'])
#print(lines)
a, b, c = lines.shape
print(lines.shape)
for i in range(a):
cv2.line(cdst, (lines[i][0][0], lines[i][0][1]), (lines[i][0][2], lines[i][0][3]), (0, 0, 255), 3,
cv2.LINE_AA)
cv2.imshow("detected lines", cdst)
cv2.imwrite(path + "traced.jpeg",cdst )
else: # HoughLines
if not param:
lines = cv2.HoughLines(imagebw, 1, math.pi / 180.0, 50, np.array([]), 0, 0)
else:
lines = cv2.HoughLines(imagebw, param['rho'], param['theta'], param['threshold'],np.array([]), param['srn'], param['stn'])
#print(lines)
if lines is not None:
a, b, c = lines.shape
print(lines.shape)
for i in range(a):
rho = lines[i][0][0]
theta = lines[i][0][1]
a = math.cos(theta)
b = math.sin(theta)
x0, y0 = a * rho, b * rho
pt1 = (int(x0 + 1000 * (-b)), int(y0 + 1000 * (a)))
pt2 = (int(x0 - 1000 * (-b)), int(y0 - 1000 * (a)))
cv2.line(cdst, pt1, pt2, (0, 0, 255), 3, cv2.LINE_AA)
cv2.imshow("detected lines", cdst)
cv2.imwrite(path + "traced.jpeg",cdst)
# cv2.imshow("source", src)
cv2.waitKey(0)
def render_image(self, extent, size):
ds = self.layer.gdal_dataset()
gt = ds.GetGeoTransform()
result = PIL.Image.new("RGBA", size, (0, 0, 0, 0))
# пересчитываем координаты в пикселы
off_x = int((extent[0] - gt[0]) / gt[1])
off_y = int((extent[3] - gt[3]) / gt[5])
width_x = int(((extent[2] - gt[0]) / gt[1]) - off_x)
width_y = int(((extent[1] - gt[3]) / gt[5]) - off_y)
# проверяем, чтобы пикселы не вылезали за пределы изображения
target_width, target_height = size
offset_left = offset_top = 0
# правая граница
if off_x + width_x > ds.RasterXSize:
oversize_right = off_x + width_x - ds.RasterXSize
target_width -= int(float(oversize_right) / width_x * target_width)
width_x -= oversize_right
# левая граница
if off_x < 0:
oversize_left = -off_x
offset_left = int(float(oversize_left) / width_x * target_width)
target_width -= int(float(oversize_left) / width_x * target_width)
width_x -= oversize_left
off_x = 0
# нижняя граница
if off_y + width_y > ds.RasterYSize:
oversize_bottom = off_y + width_y - ds.RasterYSize
target_height -= round(
float(oversize_bottom) / width_y * target_height)
width_y -= oversize_bottom
# верхняя граница
if off_y < 0:
oversize_top = -off_y
offset_top = int(float(oversize_top) / width_y * target_height)
target_height -= int(float(oversize_top) / width_y * target_height)
width_y -= oversize_top
off_y = 0
if target_width <= 0 or target_height <= 0:
# экстент не пересекается с экстентом изображения
# возвращаем пустую картинку
return result
band_count = ds.RasterCount
array = numpy.zeros((target_height, target_width, band_count),
numpy.uint8)
for i in range(band_count):
array[:, :,
i] = gdal_array.BandReadAsArray(ds.GetRasterBand(i + 1),
off_x, off_y, width_x,
width_y, target_width,
target_height)
wnd = PIL.Image.fromarray(array)
result.paste(wnd, (offset_left, offset_top))
return result
def scaleTile(dsquery, dstile, resampling, tile=''):
querysize = dsquery.RasterXSize
tile_size = dstile.RasterXSize
tilebands = dstile.RasterCount
if resampling == 'average':
# Function: gdal.RegenerateOverview()
for i in range(1, tilebands + 1):
# Black border around NODATA
res = gdal.RegenerateOverview(dsquery.GetRasterBand(i),
dstile.GetRasterBand(i), 'average')
if res != 0:
QgsMessageLog.logMessage(
"RegenerateOverview() failed on %s, error %d" %
(tile, res), CATEGORY, Qgis.Info)
elif resampling == 'antialias' and numpy_available:
# Scaling by PIL (Python Imaging Library) - improved Lanczos
array = numpy.zeros((querysize, querysize, tilebands), numpy.uint8)
for i in range(tilebands):
array[:, :,
i] = gdalarray.BandReadAsArray(dsquery.GetRasterBand(i + 1),
0, 0, querysize, querysize)
im = Image.fromarray(array, 'RGBA') # Always four bands
im1 = im.resize((tile_size, tile_size), Image.ANTIALIAS)
if os.path.exists(tile):
im0 = Image.open(tile)
im1 = Image.composite(im1, im0, im1)
im1.save(tile, 'PNG')
else:
if resampling == 'near':
gdal_resampling = gdal.GRA_NearestNeighbour
elif resampling == 'bilinear':
gdal_resampling = gdal.GRA_Bilinear
elif resampling == 'cubic':
gdal_resampling = gdal.GRA_Cubic
elif resampling == 'cubicspline':
gdal_resampling = gdal.GRA_CubicSpline
elif resampling == 'lanczos':
gdal_resampling = gdal.GRA_Lanczos
elif resampling == 'mode':
gdal_resampling = gdal.GRA_Mode
elif resampling == 'max':
gdal_resampling = gdal.GRA_Max
elif resampling == 'min':
gdal_resampling = gdal.GRA_Min
elif resampling == 'med':
gdal_resampling = gdal.GRA_Med
elif resampling == 'q1':
gdal_resampling = gdal.GRA_Q1
elif resampling == 'q3':
gdal_resampling = gdal.GRA_Q3
# Other algorithms are implemented by gdal.ReprojectImage().
dsquery.SetGeoTransform((0.0, tile_size / float(querysize), 0.0, 0.0,
0.0, tile_size / float(querysize)))
dstile.SetGeoTransform((0.0, 1.0, 0.0, 0.0, 0.0, 1.0))
res = gdal.ReprojectImage(dsquery, dstile, None, None, gdal_resampling)
if res != 0:
QgsMessageLog.logMessage(
"ReprojectImage() failed on %s, error %d" % (tile, res),
CATEGORY, Qgis.Info)
def Calc(calc: MaybeSequence[str], outfile: Optional[PathLikeOrStr] = None, NoDataValue: Optional[Number] = None,
type: Optional[Union[GDALDataType, str]] = None, format: Optional[str] = None,
creation_options: Optional[Sequence[str]] = None, allBands: str = '', overwrite: bool = False,
hideNoData: bool = False, projectionCheck: bool = False,
color_table: Optional[ColorTableLike] = None,
extent: Optional[Extent] = None, projwin: Optional[Union[Tuple, GeoRectangle]] = None,
user_namespace: Optional[Dict]=None,
debug: bool = False, quiet: bool = False, **input_files):
if debug:
print(f"gdal_calc.py starting calculation {calc}")
# Single calc value compatibility
if isinstance(calc, (list, tuple)):
calc = calc
else:
calc = [calc]
calc = [c.strip('"') for c in calc]
creation_options = creation_options or []
# set up global namespace for eval with all functions of gdal_array, numpy
global_namespace = {key: getattr(module, key)
for module in [gdal_array, numpy] for key in dir(module) if not key.startswith('__')}
if user_namespace:
global_namespace.update(user_namespace)
if not calc:
raise Exception("No calculation provided.")
elif not outfile and format.upper() != 'MEM':
raise Exception("No output file provided.")
if format is None:
format = GetOutputDriverFor(outfile)
if isinstance(extent, GeoRectangle):
pass
elif projwin:
if isinstance(projwin, GeoRectangle):
extent = projwin
else:
extent = GeoRectangle.from_lurd(*projwin)
elif not extent:
extent = Extent.IGNORE
else:
extent = extent_util.parse_extent(extent)
compatible_gt_eps = 0.000001
gt_diff_support = {
GT.INCOMPATIBLE_OFFSET: extent != Extent.FAIL,
GT.INCOMPATIBLE_PIXEL_SIZE: False,
GT.INCOMPATIBLE_ROTATION: False,
GT.NON_ZERO_ROTATION: False,
}
gt_diff_error = {
GT.INCOMPATIBLE_OFFSET: 'different offset',
GT.INCOMPATIBLE_PIXEL_SIZE: 'different pixel size',
GT.INCOMPATIBLE_ROTATION: 'different rotation',
GT.NON_ZERO_ROTATION: 'non zero rotation',
}
################################################################
# fetch details of input layers
################################################################
# set up some lists to store data for each band
myFileNames = [] # input filenames
myFiles = [] # input DataSets
myBands = [] # input bands
myAlphaList = [] # input alpha letter that represents each input file
myDataType = [] # string representation of the datatype of each input file
myDataTypeNum = [] # datatype of each input file
myNDV = [] # nodatavalue for each input file
DimensionsCheck = None # dimensions of the output
Dimensions = [] # Dimensions of input files
ProjectionCheck = None # projection of the output
GeoTransformCheck = None # GeoTransform of the output
GeoTransforms = [] # GeoTransform of each input file
GeoTransformDiffer = False # True if we have inputs with different GeoTransforms
myTempFileNames = [] # vrt filename from each input file
myAlphaFileLists = [] # list of the Alphas which holds a list of inputs
# loop through input files - checking dimensions
for alphas, filenames in input_files.items():
if isinstance(filenames, (list, tuple)):
# alpha is a list of files
myAlphaFileLists.append(alphas)
elif is_path_like(filenames) or isinstance(filenames, gdal.Dataset):
# alpha is a single filename or a Dataset
filenames = [filenames]
alphas = [alphas]
else:
# I guess this alphas should be in the global_namespace,
# It would have been better to pass it as user_namepsace, but I'll accept it anyway
global_namespace[alphas] = filenames
continue
for alpha, filename in zip(alphas * len(filenames), filenames):
if not alpha.endswith("_band"):
# check if we have asked for a specific band...
alpha_band = f"{alpha}_band"
if alpha_band in input_files:
myBand = input_files[alpha_band]
else:
myBand = 1
myF_is_ds = not is_path_like(filename)
if myF_is_ds:
myFile = filename
filename = None
else:
myFile = open_ds(filename, gdal.GA_ReadOnly)
if not myFile:
raise IOError(f"No such file or directory: '{filename}'")
myFileNames.append(filename)
myFiles.append(myFile)
myBands.append(myBand)
myAlphaList.append(alpha)
dt = myFile.GetRasterBand(myBand).DataType
myDataType.append(gdal.GetDataTypeName(dt))
myDataTypeNum.append(dt)
myNDV.append(None if hideNoData else myFile.GetRasterBand(myBand).GetNoDataValue())
# check that the dimensions of each layer are the same
myFileDimensions = [myFile.RasterXSize, myFile.RasterYSize]
if DimensionsCheck:
if DimensionsCheck != myFileDimensions:
GeoTransformDiffer = True
if extent in [Extent.IGNORE, Extent.FAIL]:
raise Exception(
f"Error! Dimensions of file {filename} ({myFileDimensions[0]:d}, "
f"{myFileDimensions[1]:d}) are different from other files "
f"({DimensionsCheck[0]:d}, {DimensionsCheck[1]:d}). Cannot proceed")
else:
DimensionsCheck = myFileDimensions
# check that the Projection of each layer are the same
myProjection = myFile.GetProjection()
if ProjectionCheck:
if projectionCheck and ProjectionCheck != myProjection:
raise Exception(
f"Error! Projection of file {filename} {myProjection} "
f"are different from other files {ProjectionCheck}. Cannot proceed")
else:
ProjectionCheck = myProjection
# check that the GeoTransforms of each layer are the same
myFileGeoTransform = myFile.GetGeoTransform(can_return_null=True)
if extent == Extent.IGNORE:
GeoTransformCheck = myFileGeoTransform
else:
Dimensions.append(myFileDimensions)
GeoTransforms.append(myFileGeoTransform)
if not GeoTransformCheck:
GeoTransformCheck = myFileGeoTransform
else:
my_gt_diff = extent_util.gt_diff(GeoTransformCheck, myFileGeoTransform, eps=compatible_gt_eps,
diff_support=gt_diff_support)
if my_gt_diff not in [GT.SAME, GT.ALMOST_SAME]:
GeoTransformDiffer = True
if my_gt_diff != GT.COMPATIBLE_DIFF:
raise Exception(
f"Error! GeoTransform of file {filename} {myFileGeoTransform} is incompatible "
f"({gt_diff_error[my_gt_diff]}), first file GeoTransform is {GeoTransformCheck}. "
f"Cannot proceed")
if debug:
print(
f"file {alpha}: {filename}, dimensions: "
f"{DimensionsCheck[0]}, {DimensionsCheck[1]}, type: {myDataType[-1]}")
# process allBands option
allBandsIndex = None
allBandsCount = 1
if allBands:
if len(calc) > 1:
raise Exception("Error! --allBands implies a single --calc")
try:
allBandsIndex = myAlphaList.index(allBands)
except ValueError:
raise Exception(f"Error! allBands option was given but Band {allBands} not found. Cannot proceed")
allBandsCount = myFiles[allBandsIndex].RasterCount
if allBandsCount <= 1:
allBandsIndex = None
else:
allBandsCount = len(calc)
if extent not in [Extent.IGNORE, Extent.FAIL] and (
GeoTransformDiffer or isinstance(extent, GeoRectangle)):
# mixing different GeoTransforms/Extents
GeoTransformCheck, DimensionsCheck, ExtentCheck = extent_util.calc_geotransform_and_dimensions(
GeoTransforms, Dimensions, extent)
if GeoTransformCheck is None:
raise Exception("Error! The requested extent is empty. Cannot proceed")
for i in range(len(myFileNames)):
temp_vrt_filename, temp_vrt_ds = extent_util.make_temp_vrt(myFiles[i], ExtentCheck)
myTempFileNames.append(temp_vrt_filename)
myFiles[i] = None # close original ds
myFiles[i] = temp_vrt_ds # replace original ds with vrt_ds
# update the new precise dimensions and gt from the new ds
GeoTransformCheck = temp_vrt_ds.GetGeoTransform()
DimensionsCheck = [temp_vrt_ds.RasterXSize, temp_vrt_ds.RasterYSize]
temp_vrt_ds = None
################################################################
# set up output file
################################################################
# open output file exists
if outfile and os.path.isfile(outfile) and not overwrite:
if allBandsIndex is not None:
raise Exception("Error! allBands option was given but Output file exists, must use --overwrite option!")
if len(calc) > 1:
raise Exception(
"Error! multiple calc options were given but Output file exists, must use --overwrite option!")
if debug:
print(f"Output file {outfile} exists - filling in results into file")
myOut = open_ds(outfile, gdal.GA_Update)
if myOut is None:
error = 'but cannot be opened for update'
elif [myOut.RasterXSize, myOut.RasterYSize] != DimensionsCheck:
error = 'but is the wrong size'
elif ProjectionCheck and ProjectionCheck != myOut.GetProjection():
error = 'but is the wrong projection'
elif GeoTransformCheck and GeoTransformCheck != myOut.GetGeoTransform(can_return_null=True):
error = 'but is the wrong geotransform'
else:
error = None
if error:
raise Exception(
f"Error! Output exists, {error}. Use the --overwrite option "
f"to automatically overwrite the existing file")
myOutB = myOut.GetRasterBand(1)
myOutNDV = myOutB.GetNoDataValue()
myOutType = myOutB.DataType
else:
if outfile:
# remove existing file and regenerate
if os.path.isfile(outfile):
os.remove(outfile)
# create a new file
if debug:
print(f"Generating output file {outfile}")
else:
outfile = ''
# find data type to use
if not type:
# use the largest type of the input files
myOutType = max(myDataTypeNum)
else:
myOutType = type
if isinstance(myOutType, str):
myOutType = gdal.GetDataTypeByName(myOutType)
# create file
myOutDrv = gdal.GetDriverByName(format)
myOut = myOutDrv.Create(
os.fspath(outfile), DimensionsCheck[0], DimensionsCheck[1], allBandsCount,
myOutType, creation_options)
# set output geo info based on first input layer
if not GeoTransformCheck:
GeoTransformCheck = myFiles[0].GetGeoTransform(can_return_null=True)
if GeoTransformCheck:
myOut.SetGeoTransform(GeoTransformCheck)
if not ProjectionCheck:
ProjectionCheck = myFiles[0].GetProjection()
if ProjectionCheck:
myOut.SetProjection(ProjectionCheck)
if NoDataValue is None:
myOutNDV = None if hideNoData else DefaultNDVLookup[
myOutType] # use the default noDataValue for this datatype
elif isinstance(NoDataValue, str) and NoDataValue.lower() == 'none':
myOutNDV = None # not to set any noDataValue
else:
myOutNDV = NoDataValue # use the given noDataValue
for i in range(1, allBandsCount + 1):
myOutB = myOut.GetRasterBand(i)
if myOutNDV is not None:
myOutB.SetNoDataValue(myOutNDV)
if color_table:
# set color table and color interpretation
if is_path_like(color_table):
color_table = get_color_table(color_table)
myOutB.SetRasterColorTable(color_table)
myOutB.SetRasterColorInterpretation(gdal.GCI_PaletteIndex)
myOutB = None # write to band
myOutTypeName = gdal.GetDataTypeName(myOutType)
if debug:
print(f"output file: {outfile}, dimensions: {myOut.RasterXSize}, {myOut.RasterYSize}, type: {myOutTypeName}")
################################################################
# find block size to chop grids into bite-sized chunks
################################################################
# use the block size of the first layer to read efficiently
myBlockSize = myFiles[0].GetRasterBand(myBands[0]).GetBlockSize()
# find total x and y blocks to be read
nXBlocks = (int)((DimensionsCheck[0] + myBlockSize[0] - 1) / myBlockSize[0])
nYBlocks = (int)((DimensionsCheck[1] + myBlockSize[1] - 1) / myBlockSize[1])
myBufSize = myBlockSize[0] * myBlockSize[1]
if debug:
print(f"using blocksize {myBlockSize[0]} x {myBlockSize[1]}")
# variables for displaying progress
ProgressCt = -1
ProgressMk = -1
ProgressEnd = nXBlocks * nYBlocks * allBandsCount
################################################################
# start looping through each band in allBandsCount
################################################################
for bandNo in range(1, allBandsCount + 1):
################################################################
# start looping through blocks of data
################################################################
# store these numbers in variables that may change later
nXValid = myBlockSize[0]
nYValid = myBlockSize[1]
# loop through X-lines
for X in range(0, nXBlocks):
# in case the blocks don't fit perfectly
# change the block size of the final piece
if X == nXBlocks - 1:
nXValid = DimensionsCheck[0] - X * myBlockSize[0]
# find X offset
myX = X * myBlockSize[0]
# reset buffer size for start of Y loop
nYValid = myBlockSize[1]
myBufSize = nXValid * nYValid
# loop through Y lines
for Y in range(0, nYBlocks):
ProgressCt += 1
if 10 * ProgressCt / ProgressEnd % 10 != ProgressMk and not quiet:
ProgressMk = 10 * ProgressCt / ProgressEnd % 10
from sys import version_info
if version_info >= (3, 0, 0):
exec('print("%d.." % (10*ProgressMk), end=" ")')
else:
exec('print 10*ProgressMk, "..",')
# change the block size of the final piece
if Y == nYBlocks - 1:
nYValid = DimensionsCheck[1] - Y * myBlockSize[1]
myBufSize = nXValid * nYValid
# find Y offset
myY = Y * myBlockSize[1]
# create empty buffer to mark where nodata occurs
myNDVs = None
# make local namespace for calculation
local_namespace = {}
val_lists = defaultdict(list)
# fetch data for each input layer
for i, Alpha in enumerate(myAlphaList):
# populate lettered arrays with values
if allBandsIndex is not None and allBandsIndex == i:
myBandNo = bandNo
else:
myBandNo = myBands[i]
myval = gdal_array.BandReadAsArray(myFiles[i].GetRasterBand(myBandNo),
xoff=myX, yoff=myY,
win_xsize=nXValid, win_ysize=nYValid)
if myval is None:
raise Exception(f'Input block reading failed from filename {filename[i]}')
# fill in nodata values
if myNDV[i] is not None:
# myNDVs is a boolean buffer.
# a cell equals to 1 if there is NDV in any of the corresponding cells in input raster bands.
if myNDVs is None:
# this is the first band that has NDV set. we initializes myNDVs to a zero buffer
# as we didn't see any NDV value yet.
myNDVs = numpy.zeros(myBufSize)
myNDVs.shape = (nYValid, nXValid)
myNDVs = 1 * numpy.logical_or(myNDVs == 1, myval == myNDV[i])
# add an array of values for this block to the eval namespace
if Alpha in myAlphaFileLists:
val_lists[Alpha].append(myval)
else:
local_namespace[Alpha] = myval
myval = None
for lst in myAlphaFileLists:
local_namespace[lst] = val_lists[lst]
# try the calculation on the array blocks
this_calc = calc[bandNo - 1 if len(calc) > 1 else 0]
try:
myResult = eval(this_calc, global_namespace, local_namespace)
except:
print(f"evaluation of calculation {this_calc} failed")
raise
# Propagate nodata values (set nodata cells to zero
# then add nodata value to these cells).
if myNDVs is not None and myOutNDV is not None:
myResult = ((1 * (myNDVs == 0)) * myResult) + (myOutNDV * myNDVs)
elif not isinstance(myResult, numpy.ndarray):
myResult = numpy.ones((nYValid, nXValid)) * myResult
# write data block to the output file
myOutB = myOut.GetRasterBand(bandNo)
if gdal_array.BandWriteArray(myOutB, myResult, xoff=myX, yoff=myY) != 0:
raise Exception('Block writing failed')
myOutB = None # write to band
# remove temp files
for idx, tempFile in enumerate(myTempFileNames):
myFiles[idx] = None
os.remove(tempFile)
gdal.ErrorReset()
myOut.FlushCache()
if gdal.GetLastErrorMsg() != '':
raise Exception('Dataset writing failed')
if not quiet:
print("100 - Done")
return myOut
def makeNDVIandEVI(state_1, rsb01, rsb02, rsb03, names, NdviPath, EviPath):
areas = {
'HuangHuaiHai': [482000, 3189000, 1574000, 4662000],
'ChangJiangZhongXiaYou': [-704000, 2596000, 1574000, 3841000],
'DongBei': [1033000, 4267000, 2207000, 5922000]
}
outputBounds = [-704000, 2596000, 2207000, 5922000]
if (state_1 is None):
print("state_1 is None")
return
if (rsb01 is None):
return
if (rsb02 is None):
return
if (rsb03 is None):
return
#state1是对遥感的像元质量检测，去掉没有云的
state1 = gdal_array.BandReadAsArray(state_1.GetRasterBand(1))
b3 = gdal_array.BandReadAsArray(rsb03.GetRasterBand(1))
b2 = gdal_array.BandReadAsArray(rsb02.GetRasterBand(1))
b1 = gdal_array.BandReadAsArray(rsb01.GetRasterBand(1))
## NDVI
ndvi = ((b2 - b1) * 1.0) / ((b2 + b1) * 1.0)
ndvi[numpy.isnan(ndvi)] = -9999
ndvi[numpy.isinf(ndvi)] = -9999
ndvi[ndvi > 1] = -9999
ndvi[ndvi < -1] = -9999
nodata = rsb02.GetRasterBand(1).GetNoDataValue()
ndvi[b2 == nodata] = -9999
nodata = rsb01.GetRasterBand(1).GetNoDataValue()
ndvi[b1 == nodata] = -9999
try:
state1_10 = state1 << 10
for i in range(0, 3600):
for j in range(0, 3600):
if state1_10[i][j] == 0:
ndvi[2 * i][2 * j] = -9999
ndvi[2 * i][2 * j + 1] = -9999
ndvi[2 * i + 1][2 * j] = -9999
ndvi[2 * i + 1][2 * j + 1] = -9999
except Exception as e:
print(e)
outNdvi = gdal_array.SaveArray(ndvi, "fgf", format="MEM", prototype=rsb01)
NdviFileName = 'MOD09GA.%s.%s.%s.tif' % (names[1], names[3], 'ndvi')
NdviFile = os.path.join(NdviPath, NdviFileName)
gdal.Warp(
NdviFile,
outNdvi,
outputBounds=outputBounds,
xRes=1000,
yRes=1000,
srcNodata=-9999,
dstNodata=-9999,
outputType=gdal.GDT_Float32,
dstSRS=
"+proj=aea +ellps=WGS84 +datum=WGS84 +lon_0=105 +lat_1=25 +lat_2=47 +units=m +"
)
#only huanghuaihai
areaNdviFileName = 'MOD09GA.%s.%s.%s.tif' % (names[1], names[3],
'huanghuaihai.ndvi')
areaNdviFile = os.path.join(NdviPath, areaNdviFileName)
gdal.Warp(areaNdviFile,
NdviFile,
outputBounds=[482000, 3189000, 1574000, 4662000])
#only dongbei
'''
def run_loraccs(self, ref_img_band, tgt_img_band, band_num, band_name,
band_max_spectra, loess_frac, tgt_img_fp, outdir):
'''
Runs the LORACCS method.
'''
os.chdir(outdir)
# Plot 2d histogram
index = (ref_img_band>0)&(tgt_img_band>0)
ref_img_band_sub = ref_img_band[index]
tgt_img_band_sub = tgt_img_band[index]
plt.hist2d(tgt_img_band_sub, ref_img_band_sub, bins=200, cmin = 5, cmap=plt.cm.jet, )
plt.colorbar()
plt.title('%s Band 2D Histogram' %band_name)
plt.xlabel('Target')
plt.ylabel('Reference')
save_fig = '%s_2d_hist.png' %band_name
plt.savefig(save_fig)
plt.show()
### Extract spectral values into a dict
# Get unique values from target image
tgt_uniq = np.unique(tgt_img_band)
counts_dict = dict()
for uniq in tgt_uniq:
counts_dict[uniq] = []
img_rows = range(0, tgt_img_band.shape[0])
img_row_pixel = range(0, tgt_img_band.shape[1])
for band_row in img_rows: # iterate through rows
for pixel in img_row_pixel: # iterate through pixels
tgt_val = tgt_img_band[band_row][pixel]
ref_val = ref_img_band[band_row][pixel]
if tgt_val != 0:
if ref_val != 0:
# Add value to the dict
values = counts_dict[tgt_val]
try:
values.append(ref_val)
except:
values = ref_val
else:
continue
# Generate stats
for uniq in tgt_uniq:
values = np.array(counts_dict[uniq])
pixels = len(values)
# Subset out values to get rid of outliers
sub = np.sort(values)
sub = sub[sub < band_max_spectra]
val_sub = sub[int(len(sub) * .025) : int(len(sub) * .975)]
try:
mean = np.mean(val_sub)
std = np.std(val_sub)
except:
print('Exception used')
mean = np.mean(counts_dict[uniq])
std = np.std(counts_dict[uniq])
new_dict = {'values' : counts_dict[uniq], 'mean' : mean, 'std' : std, 'pixels' : pixels}
counts_dict[uniq] = new_dict
# Create pandas DataFrame of values
spec_vals = tgt_uniq
mean = []
std = []
pix = []
for uniq in tgt_uniq:
mean.append(counts_dict[uniq]['mean'])
std.append(counts_dict[uniq]['std'])
pix.append(counts_dict[uniq]['pixels'])
stats_df = pd.DataFrame()
stats_df['Spec_vals'] = spec_vals
stats_df['Mean'] = mean
stats_df['Std'] = std
stats_df['Pixels'] = pix
# Remove all NaN
stats_df = stats_df.fillna(0)
stats_df_valid = stats_df[stats_df.Mean != 0]
# Remove entries with pixel count less than 6
stats_df_valid = stats_df_valid[stats_df_valid.Pixels > 5]
### Create model
# Set up params for LOESS
x = stats_df_valid.Spec_vals.values
y = stats_df_valid.Mean.values
# Run LOESS
xout, yout, wout = loess_1d(x, y, frac=loess_frac, degree=2, rotate=False)
# Save values into the dataframe
stats_df_valid['Mean_LOESS'] = yout
# Remove any bad LOESS values (rare)
stats_df_valid = stats_df_valid[stats_df_valid['Mean_LOESS'].values < band_max_spectra].copy()
stats_df_valid = stats_df_valid[stats_df_valid['Mean_LOESS'].values != 0].copy()
# Save the data to CSV
stats_df_valid.to_csv('%s_df.csv' %band_name, index=False)
# Fill gaps in spectra
min_spectra = min(stats_df_valid.Spec_vals.values)
max_spectra = max(stats_df_valid.Spec_vals.values)
if max_spectra > band_max_spectra:
reasonable_spec_vals = stats_df_valid[stats_df_valid['Spec_vals'] < band_max_spectra]
max_spectra = reasonable_spec_vals['Spec_vals'].values[-1]
print('Maxiumum spectral value being set to: ', max_spectra)
spectral_range = range(int(min_spectra), int(max_spectra+1))
full_spectra = pd.DataFrame()
full_spectra['Spec_vals'] = spectral_range
full_spectra = full_spectra.merge(stats_df_valid, how='left', on='Spec_vals')
full_spectra.drop(['Std'], axis=1, inplace=True)
full_spectra.rename(columns={'Mean':'Org_Mean'}, inplace=True)
full_spectra['Missing'] = pd.isna(full_spectra['Mean_LOESS']) # Identify missing spectral values
all_y_values = []
# Predict missing spectral values
for item in range(0, len(full_spectra)):
if full_spectra['Missing'].iloc[item] == True:
# Find nearest values on either side
invalid_before_value = True
n = item
while invalid_before_value == True:
n = n-1
invalid_before_value = full_spectra['Missing'].iloc[n]
x1 = full_spectra['Spec_vals'].iloc[n]
y1 = full_spectra['Mean_LOESS'].iloc[n]
n = item
invalid_after_value = True
while invalid_after_value == True:
n = n+1
invalid_after_value = full_spectra['Missing'].iloc[n]
x2 = full_spectra['Spec_vals'].iloc[n]
y2 = full_spectra['Mean_LOESS'].iloc[n]
# Predict new spectra value using the equation of a line between points
new_x = full_spectra['Spec_vals'].iloc[item]
new_y = self.get_new_spec_val(x1, x2, y1, y2, new_x)
else:
new_y = full_spectra['Mean_LOESS'].iloc[item]
all_y_values.append(new_y)
full_spectra['Filled_LOESS']=all_y_values
full_spectra.fillna(0, inplace=True)
### Write full spectra data frame to csv
full_spectra.to_csv('%s full spectra.csv' %band_name, index=False)
### Plot result of LORACCS along with histogram
fig, ax = plt.subplots(nrows=1, figsize=(6,4))
for_plot = full_spectra.copy()
for_plot = for_plot[for_plot['Missing'] == False]
x=for_plot['Spec_vals'].values
y1=for_plot['Filled_LOESS'].values
y2=for_plot['Pixels'].values
y3=for_plot['Org_Mean'].values
# Plot histogram
ax.bar(x, y2, width=1, color='lightgray')
gray_patch = mpatches.Patch(color='lightgray', label='Histogram')
# Set plot to have two y axes
ax2 = ax.twinx()
# Original target values as a scatterplot
ax2.scatter(x, y3, color='tab:gray', marker='.', label='Mean Reference')
#LORACCS regression line
ax2.plot(x, y1, color='tab:orange', label='LORACCS Target', linewidth=2)
# Fix tick marks
ylabs = ax2.get_yticks()
ax2.yaxis.tick_left()
ax2.set_yticklabels(ylabs, fontsize=13)
ax2.yaxis.set_major_formatter(FormatStrFormatter('%.0f'))
y2labs = ax.get_yticks()
ax.yaxis.tick_right()
ax.set_yticklabels(y2labs, fontsize=13)
ax.yaxis.set_major_formatter(FormatStrFormatter('%.0f'))
xlabs = ax2.get_xticks()
ax2.set_xticklabels(xlabs, fontsize=13)
ax2.xaxis.set_major_formatter(FormatStrFormatter('%.0f'))
ax.set_title('LORACCS Model: %s Band' %band_name, fontsize=20)
ax.set_xlabel('Target Spectral Values', fontsize=15)
ax.yaxis.set_label_position('right')
ax.set_ylabel('Reference Histogram', fontsize=15)
ax2.yaxis.set_label_position('left')
ax2.set_ylabel('Reference Spectral Values', fontsize=15)
ax.legend(fontsize=12, loc='upper left', handles=[gray_patch])
ax2.legend(fontsize=12, loc='lower right')
save_fig = '%s_LORACCS_full_spectra_plot.png' %band_name
plt.savefig(save_fig)
plt.show()
### Transform image using filled-in LORACCS function
# Read in target image
full_tgt_img = gdal.Open(tgt_img_fp)
# Get bands
band_data = full_tgt_img.GetRasterBand(band_num)
# Read in as numpy arrays
data = gdal_array.BandReadAsArray(band_data)
spec_vals_dict = dict(zip(full_spectra.Spec_vals, full_spectra.Filled_LOESS))
# Change the data type in preparation for changing values
data = data.astype('float')
# Loop through spectral values, replace with new value / 100000. Division
# necessary so already replaced values are not overwritten
for spec_val in spec_vals_dict:
data[data == spec_val] = spec_vals_dict[spec_val] / 100000
# Multiply by 100000 to restore proper values, return dtype
data = data*100000
data = data.astype('uint16')
return data # Returns band array transformed by the LORACCS method
def render_image(self, extent, size):
ds = self.parent.gdal_dataset()
gt = ds.GetGeoTransform()
result = PIL.Image.new("RGBA", size, (0, 0, 0, 0))
# recalculate coords in pixels
off_x = int((extent[0] - gt[0]) / gt[1])
off_y = int((extent[3] - gt[3]) / gt[5])
width_x = int(((extent[2] - gt[0]) / gt[1]) - off_x)
width_y = int(((extent[1] - gt[3]) / gt[5]) - off_y)
width_x = max(width_x, 1)
width_y = max(width_y, 1)
# check that pixels are not outside of image extent
target_width, target_height = size
offset_left = offset_top = 0
# right boundary
if off_x + width_x > ds.RasterXSize:
oversize_right = off_x + width_x - ds.RasterXSize
target_width -= int(float(oversize_right) / width_x * target_width)
width_x -= oversize_right
# left boundary
if off_x < 0:
oversize_left = -off_x
offset_left = int(float(oversize_left) / width_x * target_width)
target_width -= int(float(oversize_left) / width_x * target_width)
width_x -= oversize_left
off_x = 0
# bottom boundary
if off_y + width_y > ds.RasterYSize:
oversize_bottom = off_y + width_y - ds.RasterYSize
target_height -= int(
float(oversize_bottom) / width_y * target_height)
width_y -= oversize_bottom
# top boundary
if off_y < 0:
oversize_top = -off_y
offset_top = int(float(oversize_top) / width_y * target_height)
target_height -= int(float(oversize_top) / width_y * target_height)
width_y -= oversize_top
off_y = 0
if target_width <= 0 or target_height <= 0:
# extent doesn't intersect with image extent
# return empty image
return result
band_count = ds.RasterCount
array = numpy.zeros((target_height, target_width, band_count),
numpy.uint8)
for i in range(band_count):
array[:, :,
i] = gdal_array.BandReadAsArray(ds.GetRasterBand(i + 1),
off_x, off_y, width_x,
width_y, target_width,
target_height)
wnd = PIL.Image.fromarray(array)
result.paste(wnd, (offset_left, offset_top))
return result
def get_NRMSE (self, band_list, band_names, outdir):
'''
Returns a dataframe of mean-normalized RMSE values. Used to assess
quality of a LORACCS-normalized image as compared to the original imagery.
'''
os.chdir(outdir)
reference_file = 'Ref_assessment_pixels.tif'
org_file = 'Tgt_assessment_pixels.tif'
loraccs_file = 'LORACCS_assessment_pixels.tif'
# Set up dataframe for NRMSE values
nrmse_df = pd.DataFrame(index=band_list)
nrmse_df['Band Mean'] = None
nrmse_df['Original NRMSE'] = None
nrmse_df['LORACCS NRMSE'] = None
# Get data and calculate NRMSE
ref_img = gdal.Open(reference_file)
org_img = gdal.Open(org_file)
loraccs_img = gdal.Open(loraccs_file)
for num, band in enumerate(band_list):
band_num = band
band_name = band_names[num]
ref_data = ref_img.GetRasterBand(band)
org_data = org_img.GetRasterBand(band)
loraccs_data = loraccs_img.GetRasterBand(band)
# Read in as numpy array
ref_img_band = gdal_array.BandReadAsArray(ref_data)
org_img_band = gdal_array.BandReadAsArray(org_data)
loraccs_img_band = gdal_array.BandReadAsArray(loraccs_data)
# Select values in array with data
index = (ref_img_band>0)&(org_img_band>0)
ref_img_band = np.array(ref_img_band[index])
org_img_band = np.array(org_img_band[index])
loraccs_img_band = np.array(loraccs_img_band[index])
ref_img_band.ravel()
org_img_band.ravel()
loraccs_img_band.ravel()
# Get band mean to use for scaling
band_mean = np.mean(ref_img_band)
nrmse_df['Band Mean'][band] = band_mean
pix_dif_array_org = abs(np.subtract((ref_img_band.astype(np.int16)),
(org_img_band.astype(np.int16))))
pix_dif_array_lor = abs(np.subtract((ref_img_band.astype(np.int16)),
(loraccs_img_band.astype(np.int16))))
pix_dif_org = pix_dif_array_org.ravel()
pix_dif_lor = pix_dif_array_lor.ravel()
pix_dif_org.sort()
pix_dif_lor.sort()
pix_dif_org_test = pix_dif_org[int(len(pix_dif_org) * .05) : int(len(pix_dif_org) * .95)]
pix_dif_lor_test = pix_dif_lor[int(len(pix_dif_lor) * .05) : int(len(pix_dif_lor) * .95)]
# Scale by band mean
scaled_pix_dif_org_test = pix_dif_org_test / band_mean
scaled_pix_dif_lor_test = pix_dif_lor_test / band_mean
pix_org_res_sq = abs(np.square(scaled_pix_dif_org_test))
pix_lor_res_sq = abs(np.square(scaled_pix_dif_lor_test))
pix_org_res_ave = abs(np.mean(pix_org_res_sq))
pix_lor_res_ave = abs(np.mean(pix_lor_res_sq))
NRMSE_org = np.sqrt(pix_org_res_ave)
NRMSE_lor = np.sqrt(pix_lor_res_ave)
nrmse_df['Original NRMSE'][band] = NRMSE_org
nrmse_df['LORACCS NRMSE'][band] = NRMSE_lor
nrmse_df['Pixel_Cnt'] = len(pix_dif_org)
nrmse_df.to_csv('NRMSE_per_band.csv')
return nrmse_df
def renderTile(self, tile):
bounds = tile.bounds()
im = None
tile_offset_left = 0
tile_offset_top = 0
if not (bounds[2] < self.data_extent[0]
or bounds[0] > self.data_extent[2]
or bounds[3] < self.data_extent[1]
or bounds[1] > self.data_extent[3]):
target_size = tile.size()
off_x = int(
(bounds[0] - self.geo_transform[0]) / self.geo_transform[1])
off_y = int(
(bounds[3] - self.geo_transform[3]) / self.geo_transform[5])
width_x = int((
(bounds[2] - self.geo_transform[0]) / self.geo_transform[1]) -
off_x)
width_y = int((
(bounds[1] - self.geo_transform[3]) / self.geo_transform[5]) -
off_y)
# Prevent from reading off the sides of an image
if off_x + width_x > self.ds.RasterXSize:
oversize_right = off_x + width_x - self.ds.RasterXSize
target_size = [
target_size[0] -
int(float(oversize_right) / width_x * target_size[0]),
target_size[1]
]
width_x = self.ds.RasterXSize - off_x
if off_x < 0:
oversize_left = -off_x
tile_offset_left = int(
float(oversize_left) / width_x * target_size[0])
target_size = [
target_size[0] -
int(float(oversize_left) / width_x * target_size[0]),
target_size[1],
]
width_x += off_x
off_x = 0
if off_y + width_y > self.ds.RasterYSize:
oversize_bottom = off_y + width_y - self.ds.RasterYSize
target_size = [
target_size[0], target_size[1] -
round(float(oversize_bottom) / width_y * target_size[1])
]
width_y = self.ds.RasterYSize - off_y
if off_y < 0:
oversize_top = -off_y
tile_offset_top = int(
float(oversize_top) / width_y * target_size[1])
target_size = [
target_size[0],
target_size[1] -
int(float(oversize_top) / width_y * target_size[1]),
]
width_y += off_y
off_y = 0
bands = self.ds.RasterCount
empty_array = numpy.zeros(shape=(target_size[1], target_size[0]),
dtype=numpy.uint8)
rgba_array = numpy.zeros((target_size[1], target_size[0], 4),
dtype=numpy.uint8)
for i in range(bands):
band = self.ds.GetRasterBand(i + 1)
color_interp = band.GetColorInterpretation()
band_array = gdalarray.BandReadAsArray(band, off_x, off_y,
width_x, width_y,
target_size[0],
target_size[1],
empty_array)
if color_interp == gdal.GCI_PaletteIndex:
color_table = band.GetRasterColorTable()
if color_table:
color_table_size = color_table.GetCount()
lookup = [
numpy.arange(color_table_size),
numpy.arange(color_table_size),
numpy.arange(color_table_size),
numpy.ones(color_table_size) * 255
]
for color_i in range(color_table_size):
entry = color_table.GetColorEntry(color_i)
for color_ii in range(4):
lookup[color_ii][color_i] = entry[color_ii]
else:
lookup = []
for band_row in range(len(band_array)):
for band_col in range(len(band_array[band_row])):
color_pixel = numpy.zeros(shape=(4, ),
dtype=numpy.uint8)
for channel in range(4):
color_pixel[channel] = lookup[channel][
band_array[band_row][band_col]]
rgba_array[band_row, band_col] = color_pixel
elif color_interp in [
gdal.GCI_RedBand, gdal.GCI_GreenBand,
gdal.GCI_BlueBand, gdal.GCI_AlphaBand
]:
index = {
gdal.GCI_RedBand: 0,
gdal.GCI_GreenBand: 1,
gdal.GCI_BlueBand: 2,
gdal.GCI_AlphaBand: 3,
}
for band_row in range(len(band_array)):
for band_col in range(len(band_array[band_row])):
rgba_array[band_row, band_col, 3] = 255 # TODO
rgba_array[band_row, band_col,
index[color_interp]] = band_array[
band_row][band_col]
else:
for band_row in range(len(band_array)):
for band_col in range(len(band_array[band_row])):
rgba_array[band_row, band_col,
0] = band_array[band_row][band_col]
rgba_array[band_row, band_col,
1] = band_array[band_row][band_col]
rgba_array[band_row, band_col,
2] = band_array[band_row][band_col]
rgba_array[band_row, band_col, 3] = 255
im = Image.fromarray(rgba_array)
big = Image.new("RGBA", tile.size(), (0, 0, 0, 0))
if im:
big.paste(im, (tile_offset_left, tile_offset_top))
buf = StringIO.StringIO()
big.save(buf, self.extension)
buf.seek(0)
tile.data = buf.read()
return tile.data
def makeNDVIEVI16(satellitename, rb1, rb2, names, ndvi16path, evi16path):
if (rb1 is None):
return
if (rb2 is None):
return
'''
areas = {'HuangHuaiHai': [482000, 3189000, 1574000, 4662000],
'ChangJiangZhongXiaYou': [-704000, 2596000, 1574000, 3841000],
'DongBei': [1033000, 4267000, 2207000, 5922000]};
'''
areas = {
'HuangHuaiHai': [482000, 3189000, 1574000, 4662000],
'DongBei': [1033000, 4267000, 2207000, 5922000]
}
########make NDVI file###########
outputBounds = [-704000, 2596000, 2207000, 5922000]
NDVI16FileName = '%s.%s.%s.%s.tif' % (satellitename, names[1], names[3],
"ndvi")
NDVIFile = os.path.join(ndvi16path, NDVI16FileName)
gdal.Warp(
NDVIFile,
rb1,
outputBounds=outputBounds,
xRes=1000,
yRes=1000,
srcNodata=-9999,
dstNodata=-9999,
outputType=gdal.GDT_Float32,
dstSRS=
"+proj=aea +ellps=WGS84 +datum=WGS84 +lon_0=105 +lat_1=25 +lat_2=47 +units=m +"
)
for name, bound in areas.items():
areaLSTFileName = '%s.%s.%s.%s.%s.tif' % (satellitename, names[1],
names[3], name, "ndvi")
areaLSTFile = os.path.join(ndvi16path, areaLSTFileName)
gdal.Warp(areaLSTFile, NDVIFile, outputBounds=bound)
##########make evi file#################
bandevi = gdal_array.BandReadAsArray(rb2.GetRasterBand(1))
bandevi[numpy.isnan(bandevi)] = -9999
bandevi[numpy.isinf(bandevi)] = -9999
bandevi = bandevi / 10000.0
bandevi[bandevi > 2] = -9999
bandevi[bandevi < -1] = -9999
nodata = rb2.GetRasterBand(1).GetNoDataValue()
bandevi[bandevi == nodata] = -9999
outevi = gdal_array.SaveArray(bandevi, "fgf", format="MEM", prototype=rb2)
evi16filename = '%s.%s.%s.%s.tif' % (satellitename, names[1], names[3],
"evi")
eVIFile = os.path.join(evi16path, evi16filename)
gdal.Warp(
eVIFile,
outevi,
outputBounds=outputBounds,
xRes=1000,
yRes=1000,
srcNodata=-9999,
dstNodata=-9999,
outputType=gdal.GDT_Float32,
dstSRS=
"+proj=aea +ellps=WGS84 +datum=WGS84 +lon_0=105 +lat_1=25 +lat_2=47 +units=m +"
)
for name, bound in areas.items():
areaLSTFileName = '%s.%s.%s.%s.%s.tif' % (satellitename, names[1],
names[3], name, "evi")
areaLSTFile = os.path.join(evi16path, areaLSTFileName)
gdal.Warp(areaLSTFile, eVIFile, outputBounds=bound)
def _get_map(obj, params, request):
p_layers = params['LAYERS'].split(',')
p_bbox = _validate_bbox([float(v) for v in params['BBOX'].split(',', 3)])
p_width = int(params['WIDTH'])
p_height = int(params['HEIGHT'])
p_format = params.get('FORMAT', IMAGE_FORMAT.PNG)
p_style = params.get('STYLES')
p_bgcolor = params.get('BGCOLOR')
p_transparent = params.get('TRANSPARENT', 'FALSE')
p_srs = params.get('SRS', params.get('CRS'))
if p_format not in IMAGE_FORMAT.enum:
raise ValidationError("Invalid FORMAT parameter.",
data=dict(code="InvalidFormat"))
if p_style and not (p_style == "," * (len(p_layers) - 1)):
raise ValidationError("Style not found.",
data=dict(code="StyleNotDefined"))
if p_srs is None:
raise ValidationError(message="CRS/SRS parameter required.")
if p_bgcolor:
r, g, b = _validate_bgcolor(p_bgcolor)
bgcolor = (r, g, b)
else:
bgcolor = (255, 255, 255)
if p_transparent.upper() == 'TRUE':
img_mode = 'RGBA'
bgcolor = bgcolor + (0, )
else:
img_mode = 'RGB'
p_size = (p_width, p_height)
img = Image.new(img_mode, p_size, bgcolor)
try:
epsg, axis_sy = parse_srs(p_srs)
except SRSParseError as e:
raise ValidationError(message=str(e), data=dict(code="InvalidSRS"))
try:
srs = SRS.filter_by(id=epsg).one()
except NoResultFound:
raise ValidationError(message="SRS (id=%d) not found." % epsg,
data=dict(code="InvalidSRS"))
def scale(delta, img_px):
dpi = 96
img_inch = float(img_px) / dpi
img_m = img_inch * 0.0254
return delta / img_m
xmin, ymin, xmax, ymax = p_bbox
if srs.is_geographic:
distance = geographic_distance(xmin, ymin, xmax, ymax)
else:
distance = xmax - xmin
w_scale = scale(distance, p_width)
for lname in p_layers:
lobj = layer_by_keyname(obj, lname)
res = lobj.resource
request.resource_permission(DataScope.read, res)
if (lobj.min_scale_denom is None or lobj.min_scale_denom >= w_scale) and \
(lobj.max_scale_denom is None or w_scale >= lobj.max_scale_denom):
req = res.render_request(res.srs)
# Do not use foreign SRS as it does not work correctly yet
if srs.id == res.srs.id:
limg = req.render_extent(p_bbox, p_size)
else:
mem = gdal.GetDriverByName('MEM')
dst_geo = (xmin, (xmax - xmin) / p_width, 0, ymax, 0,
(ymin - ymax) / p_height)
dst_ds = mem.Create('', p_width, p_height, 4, gdal.GDT_Byte)
dst_ds.SetGeoTransform(dst_geo)
vrt = gdal.AutoCreateWarpedVRT(dst_ds, srs.wkt, res.srs.wkt)
src_width = vrt.RasterXSize
src_height = vrt.RasterYSize
src_geo = vrt.GetGeoTransform()
vrt = None
src_bbox = (src_geo[0], src_geo[3] + src_geo[5] * src_height,
src_geo[0] + src_geo[1] * src_width, src_geo[3])
limg = req.render_extent(src_bbox, (src_width, src_height))
if limg is not None:
data = numpy.asarray(limg)
img_h, img_w, band_count = data.shape
src_ds = mem.Create('', src_width, src_height, band_count,
gdal.GDT_Byte)
src_ds.SetGeoTransform(src_geo)
for i in range(band_count):
bandArray = data[:, :, i]
src_ds.GetRasterBand(i + 1).WriteArray(bandArray)
gdal.ReprojectImage(src_ds, dst_ds, res.srs.wkt, srs.wkt)
array = numpy.zeros((p_height, p_width, band_count),
numpy.uint8)
for i in range(band_count):
array[:, :, i] = gdal_array.BandReadAsArray(
dst_ds.GetRasterBand(i + 1))
limg = Image.fromarray(array)
src_ds = dst_ds = None
if limg is not None:
img.paste(limg, (0, 0), limg)
buf = BytesIO()
if p_format == IMAGE_FORMAT.JPEG:
img.convert('RGB').save(buf, 'jpeg')
elif p_format == IMAGE_FORMAT.PNG:
img.save(buf, 'png', compress_level=3)
buf.seek(0)
return Response(body_file=buf, content_type=p_format)
def reclassify_raster(infile,
outfile,
operator,
threshold,
pixel_value=100,
frmt="GTiff",
silence=True):
"""
Reclassify a raster into a single value according to a threshold
:param infile: path to the input
:param outfile: path to the output
:param operator: string, possible values ">","<",">=","<="
:param threshold: threshold value
:param pixel_value: the output pixel value for pixels that pass the threshold
:param frmt: the output format, default "GTiff"
:param silence: pass any value to enable debug info
:return: None
"""
'''using this instead of gdal_calc because gdal calc didn't work
scriptpath = os.path.join(os.path.dirname(__file__), '../', "pysdss/utility/gdal_calc.py")
scriptpath = os.path.normpath(scriptpath)
params = [scriptpath,"-A",path+input,"--outfile="+path+output,"--calc='(A>0.5)'", "--debug"]
print("Reclassify raster")
out,err = utils.run_script(params, callpy=["python3"])
print("output" + out)
if err:
print("ERROR:" + err)
raise Exception("gdal grid failed")'''
if operator not in [">", "<", ">=", "<="]:
raise Exception("unknown operator")
band1 = None
ds1 = None
bandOut = None
dsOut = None
try:
if not silence:
print("opening input")
# Open the dataset
ds1 = gdal.Open(infile, gdct.GA_ReadOnly)
band1 = ds1.GetRasterBand(1)
# Read the data into numpy arrays
data1 = gdar.BandReadAsArray(band1)
# get the nodata value
nodata = band1.GetNoDataValue()
if not silence:
print("filtering input")
# The actual calculation
filter = "data1" + operator + str(threshold)
data1[eval(filter)] = pixel_value
data1[data1 != pixel_value] = nodata
if not silence:
print("output result")
# Write the out file
driver = gdal.GetDriverByName(frmt)
dsOut = driver.Create(outfile, ds1.RasterXSize, ds1.RasterYSize, 1,
band1.DataType)
gdar.CopyDatasetInfo(ds1, dsOut)
bandOut = dsOut.GetRasterBand(1)
bandOut.SetNoDataValue(nodata)
gdar.BandWriteArray(bandOut, data1)
except RuntimeError as err:
raise err
except Exception as e:
raise e
finally:
# Close the datasets
if band1: band1 = None
if ds1: ds1 = None
if bandOut: bandOut = None
if dsOut: dsOut = None
