def create_tif_mask(in_fname, out_fname): # Open the raster file raster = gdal.Open(in_fname, gdal.GA_ReadOnly) band_1 = raster.GetRasterBand(1) # Read the data as a numpy array data_1 = gdal_array.BandReadAsArray(band_1) # Create a boolean band with all 1's mask = (data_1 >= 0).astype(int) assert np.mean(mask) == 1 # Write the output mask driver = gdal.GetDriverByName("GTiff") ds_out = driver.Create(out_fname, raster.RasterXSize, raster.RasterYSize, 1, band_1.DataType) gdal_array.CopyDatasetInfo(raster, ds_out) band_out = ds_out.GetRasterBand(1) gdal_array.BandWriteArray(band_out, mask) # Close the datasets band_1 = None raster = None band_out = None ds_out = None
def render_image(self, extent, size): ds = gdal.Warp( "", self.parent.gdal_dataset(), options=gdal.WarpOptions( width=size[0], height=size[1], outputBounds=extent, format="MEM", warpOptions=['UNIFIED_SRC_NODATA=ON'], dstAlpha=True, ), ) result = PIL.Image.new("RGBA", size, (0, 0, 0, 0)) band_count = ds.RasterCount array = numpy.zeros((size[1], size[0], band_count), numpy.uint8) for i in range(band_count): array[:, :, i] = gdal_array.BandReadAsArray(ds.GetRasterBand(i + 1), ) ds = None wnd = PIL.Image.fromarray(array) result.paste(wnd) return result
def img_to_arr(input_file, dim_ordering='channels_last', dtype='float32'): """Takes in a raster file as input and converts to numpy array""" bands = [ input_file.GetRasterBand(i) for i in range(1, input_file.RasterCount + 1) ] arr = np.array([gdal_array.BandReadAsArray(band) for band in bands]).astype(dtype) if dim_ordering == 'channels_last': arr = np.transpose(arr, [1, 2, 0]) return arr
def get_clip(self, bnds): """get the part of our grid clipped by bnds (from get_bounds())""" try: rasterDat = gdal_array.BandReadAsArray(self.img.GetRasterBand(1), bnds.fcols, bnds.frows, bnds.fcolcnt, bnds.frowcnt) except: print bnds.fcols, bnds.fcole, bnds.fcolcnt, bnds.frows, \ bnds.frowe, bnds.frowcnt raise return rasterDat
def scale_query_to_tile(self, dsquery, dstile, tx, ty, tz): """Scales down query dataset to the tile dataset""" querysize = dsquery.RasterXSize tilesize = dstile.RasterXSize # Scaling by PIL (Python Imaging Library) - improved Lanczos # mod XX: to accomodate with 2 band hilshades array = numpy.zeros((querysize, querysize, 4), numpy.uint8) white=numpy.zeros((querysize, querysize), numpy.uint8) white.fill(255) #for i in [0,1,2]: # array[:,:,i] = gdalarray.BandReadAsArray(dsquery.GetRasterBand(1), 0, 0, querysize, querysize) hillshade = 255 - gdalarray.BandReadAsArray(dsquery.GetRasterBand(1), 0, 0, querysize, querysize) mask = gdalarray.BandReadAsArray(dsquery.GetRasterBand(2), 0, 0, querysize, querysize) / 255 array[:,:,3] = hillshade*mask im = Image.fromarray(array, 'RGBA') # Always four bands im1 = im.filter(ImageFilter.BLUR).resize((tilesize,tilesize), Image.ANTIALIAS) #im1.save(tilefilename,self.tiledriver) buf= StringIO.StringIO() im1.save(buf,self.tiledriver) self.store.writeImage(tx, ty, tz, buf.getvalue())
def read_image(x1, y1, x2, y2, srs): width = height = 500 ds_img = gdal.Warp('', ds, options=gdal.WarpOptions(width=width, height=height, outputBounds=(x1, y1, x2, y2), dstSRS=srs, format='MEM')) array = numpy.zeros((height, width, band_count), numpy.uint8) for i in range(band_count): band = ds_img.GetRasterBand(i + 1) array[:, :, i] = gdal_array.BandReadAsArray(band) img = Image.fromarray(array) return img
def resample_antialias(path, dsquery, dstile): querysize = dsquery.RasterXSize tile_size = dstile.RasterXSize array = numpy.zeros((querysize, querysize, 4), numpy.uint8) for i in range(dstile.RasterCount): array[:, :, i] = gdalarray.BandReadAsArray( # NOQA dsquery.GetRasterBand(i + 1), 0, 0, querysize, querysize) im = Image.fromarray(array, 'RGBA') # Always four bands im1 = im.resize((tile_size, tile_size), Image.ANTIALIAS) if path.exists(): im0 = Image.open(str(path)) im1 = Image.composite(im1, im0, im1) ensure_dir_exists(path.parent) im1.save(str(path), 'PNG')
def reclassifyRaster(raster, bins, newbinvalues): #turn raster into numpy array rasterIn = gdal.Open(raster) arr = gdal_array.BandReadAsArray(rasterIn.GetRasterBand(1)) print arr #conditional loop to reclassify arr[(bins[5] >= arr) & (arr > bins[4])] = newbinvalues[4] arr[(bins[4] >= arr) & (arr > bins[3])] = newbinvalues[3] arr[(bins[3] >= arr) & (arr > bins[2])] = newbinvalues[2] arr[(bins[2] >= arr) & (arr > bins[1])] = newbinvalues[1] arr[(bins[1] >= arr) & (arr > bins[0])] = newbinvalues[0] print arr #Array to raster rasterOut = raster.split('.')[0] + '_reclassified.TIF' array2raster(arr, rasterIn, rasterOut) return rasterOut
def single_band_to_indexed_numpy(band, nodata=-3.40282e+038, return_index=True): """ Convert a dataset band into an indexed array :param band: a gdal band :param nodata: pixel no data value :param return_index: a bool to return the array index :return: (index, r_indexed) or r_indexed , depending on return_index """ try: # convert a gdal band to a numpyarray px = gdar.BandReadAsArray(band, 0, 0, band.XSize, band.YSize) index = np.nonzero(px > nodata) # define the index r_indexed = px[index] # apply the index to the band data if return_index: return index, r_indexed else: return r_indexed except RuntimeError as err: raise err except Exception as e: raise e
def apply_index_to_single_band(band, index, return_index=False): """ Apply an existing index to a band :param band: a gdal band :param nodata: a numpy index :param return_index: a bool to return the array index :return: (index, r_indexed) or r_indexed , depending on return_index """ try: # convert a gdal band to a numpyarray px = gdar.BandReadAsArray(band, 0, 0, band.XSize, band.YSize) r_indexed = px[index] # apply the index to the band data if return_index: return index, r_indexed else: return r_indexed except RuntimeError as err: raise err except Exception as e: raise e
def band_values(self, **kwargs): """ Method to read band from arguments or from initialized raster. Will mask values defined in the band NoDataValue. :param kwargs: 'band_number': band_number to read instead of the one given with the initialize call. :return: Numpy masked array """ band_number = kwargs.get('band_number', self.band_number) band = self.file.GetRasterBand(band_number) values = np.ma.masked_values( gdal_array.BandReadAsArray(band), band.GetNoDataValue(), copy=False ) del band return values
def __init__(self, outdir, ref_img_fp, tgt_img_fp, band_list=[1, 2, 3, 4], band_names=['Blue', 'Green', 'Red', 'NIR'], max_spectra = [3000, 3000, 3000, 8000], loess_frac=0.15, delete_working_files=True): os.chdir(outdir) self.get_overlap_areas(ref_img_fp, tgt_img_fp, outdir) # Do LORACCS correction on target image ref_img = gdal.Open('Ref_training_pixels.tif') tgt_img = gdal.Open('Tgt_training_pixels.tif') loraccs_img_fp = 'LORACCS_normalized_img.tif' # File name of transformed image tgt_img_2 = rasterio.open(tgt_img_fp) # Will pull metadata from original image for LORACCS image for num, band in enumerate(band_list): # Get the band data as a numpy array ref_data = ref_img.GetRasterBand(band) tgt_data = tgt_img.GetRasterBand(band) ref_img_band = gdal_array.BandReadAsArray(ref_data) tgt_img_band = gdal_array.BandReadAsArray(tgt_data) # Do LORACCS normalization loraccs_array = self.run_loraccs(ref_img_band, tgt_img_band, band, band_names[num], max_spectra[num], loess_frac, tgt_img_fp, outdir) # Write transformed array to new image with rasterio.Env(): profile = tgt_img_2.profile profile.update(nodata=0) if num == 0: with rasterio.open(loraccs_img_fp, 'w', **profile) as dst: dst.write(loraccs_array, band) else: with rasterio.open(loraccs_img_fp, 'r+', **profile) as dst: dst.write(loraccs_array, band) # Get the diagonal pixels from the LORACCS image to run quality assessment self.get_qa_pixels('overlap.shp', loraccs_img_fp, outdir) # Generate NRMSE file self.nrmse = self.get_NRMSE(band_list, band_names, outdir) print(self.nrmse) # Print completed message print('LORACCS transformation complete. New image file can be found in the given directory.') print('File name: LORACCS_normalized_img.tif') # Delete working files if delete_working_files == True: print('Removing working files.') working_files = ['Ref_training_pixels.tif', 'QA_diag_lines.shp', 'Reference_clip_overlap.tif', 'overlap.prj', 'QA_diag_lines.cpg', 'LORACCS_image_overlap_clip.tif', 'QA_diag_lines.prj', 'overlap.shp', 'QA_diag_lines.dbf', 'Ref_assessment_pixels.tif', 'overlap.cpg', 'Tgt_assessment_pixels.tif', 'LORACCS_assessment_pixels.tif', 'Target_clip_overlap.tif', 'Tgt_training_pixels.tif', 'QA_diag_lines.shx', 'overlap.shx', 'overlap.dbf'] for name in band_names: band_files = ['%s_2d_hist.png' %name, '%s_df.csv' %name, '%s full spectra.csv' %name] working_files.extend(band_files[:]) for file in working_files: os.remove(file)
def renderTile(self, tile): import PIL.Image as PILImage import StringIO bounds = tile.bounds() im = None # If the image is entirely outside the bounds, don't bother doing anything with it: # just return an 'empty' tile. if not (bounds[2] < self.data_extent[0] or bounds[0] > self.data_extent[2] or bounds[3] < self.data_extent[1] or bounds[1] > self.data_extent[3]): tile_offset_left = tile_offset_top = 0 target_size = tile.size() off_x = int( (bounds[0] - self.geo_transform[0]) / self.geo_transform[1]) off_y = int( (bounds[3] - self.geo_transform[3]) / self.geo_transform[5]) width_x = int(( (bounds[2] - self.geo_transform[0]) / self.geo_transform[1]) - off_x) width_y = int(( (bounds[1] - self.geo_transform[3]) / self.geo_transform[5]) - off_y) # Prevent from reading off the sides of an image if off_x + width_x > self.ds.RasterXSize: oversize_right = off_x + width_x - self.ds.RasterXSize target_size = [ target_size[0] - int(float(oversize_right) / width_x * target_size[0]), target_size[1] ] width_x = self.ds.RasterXSize - off_x if off_x < 0: oversize_left = -off_x tile_offset_left = int( float(oversize_left) / width_x * target_size[0]) target_size = [ target_size[0] - int(float(oversize_left) / width_x * target_size[0]), target_size[1], ] width_x = width_x + off_x off_x = 0 if off_y + width_y > self.ds.RasterYSize: oversize_bottom = off_y + width_y - self.ds.RasterYSize target_size = [ target_size[0], target_size[1] - round(float(oversize_bottom) / width_y * target_size[1]) ] width_y = self.ds.RasterYSize - off_y if off_y < 0: oversize_top = -off_y tile_offset_top = int( float(oversize_top) / width_y * target_size[1]) target_size = [ target_size[0], target_size[1] - int(float(oversize_top) / width_y * target_size[1]), ] width_y = width_y + off_y off_y = 0 bands = self.ds.RasterCount array = numpy.zeros((target_size[1], target_size[0], bands), numpy.uint8) for i in range(bands): array[:, :, i] = gdalarray.BandReadAsArray( self.ds.GetRasterBand(i + 1), off_x, off_y, width_x, width_y, target_size[0], target_size[1]) im = PIL.Image.fromarray(array) big = PIL.Image.new("RGBA", tile.size(), (0, 0, 0, 0)) if im: big.paste(im, (tile_offset_left, tile_offset_top)) buffer = StringIO.StringIO() big.save(buffer, self.extension) buffer.seek(0) tile.data = buffer.read() return tile.data
def makelines(path,algo="HoughLinesP", param={} ): d = gdal.Open(path) band = d.GetRasterBand(1) image = gdar.BandReadAsArray(band) d=None # set to uint8 imagebw = image.astype(np.uint8, copy=False) #define the output image cdst = cv2.cvtColor(imagebw, cv2.COLOR_GRAY2BGR) # image = cv2.imread(path) # image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) # image = cv2.GaussianBlur(image, (5, 5), 0) # cv2.imshow("Blurred", image) # canny = cv2.Canny(image, 30, 150) # cv2.imshow("Canny", canny) # cv2.waitKey(0) if (algo=="HoughLinesP"): # HoughLinesP if not param: lines = cv2.HoughLinesP(imagebw, 1, math.pi / 180.0, 40, np.array([]), 50, 10) else: lines = cv2.HoughLinesP(imagebw, param['rho'], param['theta'], param['threshold'], np.array([]), param['minlinelenght'], param['maxlinegap']) #print(lines) a, b, c = lines.shape print(lines.shape) for i in range(a): cv2.line(cdst, (lines[i][0][0], lines[i][0][1]), (lines[i][0][2], lines[i][0][3]), (0, 0, 255), 3, cv2.LINE_AA) cv2.imshow("detected lines", cdst) cv2.imwrite(path + "traced.jpeg",cdst ) else: # HoughLines if not param: lines = cv2.HoughLines(imagebw, 1, math.pi / 180.0, 50, np.array([]), 0, 0) else: lines = cv2.HoughLines(imagebw, param['rho'], param['theta'], param['threshold'],np.array([]), param['srn'], param['stn']) #print(lines) if lines is not None: a, b, c = lines.shape print(lines.shape) for i in range(a): rho = lines[i][0][0] theta = lines[i][0][1] a = math.cos(theta) b = math.sin(theta) x0, y0 = a * rho, b * rho pt1 = (int(x0 + 1000 * (-b)), int(y0 + 1000 * (a))) pt2 = (int(x0 - 1000 * (-b)), int(y0 - 1000 * (a))) cv2.line(cdst, pt1, pt2, (0, 0, 255), 3, cv2.LINE_AA) cv2.imshow("detected lines", cdst) cv2.imwrite(path + "traced.jpeg",cdst) # cv2.imshow("source", src) cv2.waitKey(0)
def render_image(self, extent, size): ds = self.layer.gdal_dataset() gt = ds.GetGeoTransform() result = PIL.Image.new("RGBA", size, (0, 0, 0, 0)) # пересчитываем координаты в пикселы off_x = int((extent[0] - gt[0]) / gt[1]) off_y = int((extent[3] - gt[3]) / gt[5]) width_x = int(((extent[2] - gt[0]) / gt[1]) - off_x) width_y = int(((extent[1] - gt[3]) / gt[5]) - off_y) # проверяем, чтобы пикселы не вылезали за пределы изображения target_width, target_height = size offset_left = offset_top = 0 # правая граница if off_x + width_x > ds.RasterXSize: oversize_right = off_x + width_x - ds.RasterXSize target_width -= int(float(oversize_right) / width_x * target_width) width_x -= oversize_right # левая граница if off_x < 0: oversize_left = -off_x offset_left = int(float(oversize_left) / width_x * target_width) target_width -= int(float(oversize_left) / width_x * target_width) width_x -= oversize_left off_x = 0 # нижняя граница if off_y + width_y > ds.RasterYSize: oversize_bottom = off_y + width_y - ds.RasterYSize target_height -= round( float(oversize_bottom) / width_y * target_height) width_y -= oversize_bottom # верхняя граница if off_y < 0: oversize_top = -off_y offset_top = int(float(oversize_top) / width_y * target_height) target_height -= int(float(oversize_top) / width_y * target_height) width_y -= oversize_top off_y = 0 if target_width <= 0 or target_height <= 0: # экстент не пересекается с экстентом изображения # возвращаем пустую картинку return result band_count = ds.RasterCount array = numpy.zeros((target_height, target_width, band_count), numpy.uint8) for i in range(band_count): array[:, :, i] = gdal_array.BandReadAsArray(ds.GetRasterBand(i + 1), off_x, off_y, width_x, width_y, target_width, target_height) wnd = PIL.Image.fromarray(array) result.paste(wnd, (offset_left, offset_top)) return result
def scaleTile(dsquery, dstile, resampling, tile=''): querysize = dsquery.RasterXSize tile_size = dstile.RasterXSize tilebands = dstile.RasterCount if resampling == 'average': # Function: gdal.RegenerateOverview() for i in range(1, tilebands + 1): # Black border around NODATA res = gdal.RegenerateOverview(dsquery.GetRasterBand(i), dstile.GetRasterBand(i), 'average') if res != 0: QgsMessageLog.logMessage( "RegenerateOverview() failed on %s, error %d" % (tile, res), CATEGORY, Qgis.Info) elif resampling == 'antialias' and numpy_available: # Scaling by PIL (Python Imaging Library) - improved Lanczos array = numpy.zeros((querysize, querysize, tilebands), numpy.uint8) for i in range(tilebands): array[:, :, i] = gdalarray.BandReadAsArray(dsquery.GetRasterBand(i + 1), 0, 0, querysize, querysize) im = Image.fromarray(array, 'RGBA') # Always four bands im1 = im.resize((tile_size, tile_size), Image.ANTIALIAS) if os.path.exists(tile): im0 = Image.open(tile) im1 = Image.composite(im1, im0, im1) im1.save(tile, 'PNG') else: if resampling == 'near': gdal_resampling = gdal.GRA_NearestNeighbour elif resampling == 'bilinear': gdal_resampling = gdal.GRA_Bilinear elif resampling == 'cubic': gdal_resampling = gdal.GRA_Cubic elif resampling == 'cubicspline': gdal_resampling = gdal.GRA_CubicSpline elif resampling == 'lanczos': gdal_resampling = gdal.GRA_Lanczos elif resampling == 'mode': gdal_resampling = gdal.GRA_Mode elif resampling == 'max': gdal_resampling = gdal.GRA_Max elif resampling == 'min': gdal_resampling = gdal.GRA_Min elif resampling == 'med': gdal_resampling = gdal.GRA_Med elif resampling == 'q1': gdal_resampling = gdal.GRA_Q1 elif resampling == 'q3': gdal_resampling = gdal.GRA_Q3 # Other algorithms are implemented by gdal.ReprojectImage(). dsquery.SetGeoTransform((0.0, tile_size / float(querysize), 0.0, 0.0, 0.0, tile_size / float(querysize))) dstile.SetGeoTransform((0.0, 1.0, 0.0, 0.0, 0.0, 1.0)) res = gdal.ReprojectImage(dsquery, dstile, None, None, gdal_resampling) if res != 0: QgsMessageLog.logMessage( "ReprojectImage() failed on %s, error %d" % (tile, res), CATEGORY, Qgis.Info)
def Calc(calc: MaybeSequence[str], outfile: Optional[PathLikeOrStr] = None, NoDataValue: Optional[Number] = None, type: Optional[Union[GDALDataType, str]] = None, format: Optional[str] = None, creation_options: Optional[Sequence[str]] = None, allBands: str = '', overwrite: bool = False, hideNoData: bool = False, projectionCheck: bool = False, color_table: Optional[ColorTableLike] = None, extent: Optional[Extent] = None, projwin: Optional[Union[Tuple, GeoRectangle]] = None, user_namespace: Optional[Dict]=None, debug: bool = False, quiet: bool = False, **input_files): if debug: print(f"gdal_calc.py starting calculation {calc}") # Single calc value compatibility if isinstance(calc, (list, tuple)): calc = calc else: calc = [calc] calc = [c.strip('"') for c in calc] creation_options = creation_options or [] # set up global namespace for eval with all functions of gdal_array, numpy global_namespace = {key: getattr(module, key) for module in [gdal_array, numpy] for key in dir(module) if not key.startswith('__')} if user_namespace: global_namespace.update(user_namespace) if not calc: raise Exception("No calculation provided.") elif not outfile and format.upper() != 'MEM': raise Exception("No output file provided.") if format is None: format = GetOutputDriverFor(outfile) if isinstance(extent, GeoRectangle): pass elif projwin: if isinstance(projwin, GeoRectangle): extent = projwin else: extent = GeoRectangle.from_lurd(*projwin) elif not extent: extent = Extent.IGNORE else: extent = extent_util.parse_extent(extent) compatible_gt_eps = 0.000001 gt_diff_support = { GT.INCOMPATIBLE_OFFSET: extent != Extent.FAIL, GT.INCOMPATIBLE_PIXEL_SIZE: False, GT.INCOMPATIBLE_ROTATION: False, GT.NON_ZERO_ROTATION: False, } gt_diff_error = { GT.INCOMPATIBLE_OFFSET: 'different offset', GT.INCOMPATIBLE_PIXEL_SIZE: 'different pixel size', GT.INCOMPATIBLE_ROTATION: 'different rotation', GT.NON_ZERO_ROTATION: 'non zero rotation', } ################################################################ # fetch details of input layers ################################################################ # set up some lists to store data for each band myFileNames = [] # input filenames myFiles = [] # input DataSets myBands = [] # input bands myAlphaList = [] # input alpha letter that represents each input file myDataType = [] # string representation of the datatype of each input file myDataTypeNum = [] # datatype of each input file myNDV = [] # nodatavalue for each input file DimensionsCheck = None # dimensions of the output Dimensions = [] # Dimensions of input files ProjectionCheck = None # projection of the output GeoTransformCheck = None # GeoTransform of the output GeoTransforms = [] # GeoTransform of each input file GeoTransformDiffer = False # True if we have inputs with different GeoTransforms myTempFileNames = [] # vrt filename from each input file myAlphaFileLists = [] # list of the Alphas which holds a list of inputs # loop through input files - checking dimensions for alphas, filenames in input_files.items(): if isinstance(filenames, (list, tuple)): # alpha is a list of files myAlphaFileLists.append(alphas) elif is_path_like(filenames) or isinstance(filenames, gdal.Dataset): # alpha is a single filename or a Dataset filenames = [filenames] alphas = [alphas] else: # I guess this alphas should be in the global_namespace, # It would have been better to pass it as user_namepsace, but I'll accept it anyway global_namespace[alphas] = filenames continue for alpha, filename in zip(alphas * len(filenames), filenames): if not alpha.endswith("_band"): # check if we have asked for a specific band... alpha_band = f"{alpha}_band" if alpha_band in input_files: myBand = input_files[alpha_band] else: myBand = 1 myF_is_ds = not is_path_like(filename) if myF_is_ds: myFile = filename filename = None else: myFile = open_ds(filename, gdal.GA_ReadOnly) if not myFile: raise IOError(f"No such file or directory: '{filename}'") myFileNames.append(filename) myFiles.append(myFile) myBands.append(myBand) myAlphaList.append(alpha) dt = myFile.GetRasterBand(myBand).DataType myDataType.append(gdal.GetDataTypeName(dt)) myDataTypeNum.append(dt) myNDV.append(None if hideNoData else myFile.GetRasterBand(myBand).GetNoDataValue()) # check that the dimensions of each layer are the same myFileDimensions = [myFile.RasterXSize, myFile.RasterYSize] if DimensionsCheck: if DimensionsCheck != myFileDimensions: GeoTransformDiffer = True if extent in [Extent.IGNORE, Extent.FAIL]: raise Exception( f"Error! Dimensions of file {filename} ({myFileDimensions[0]:d}, " f"{myFileDimensions[1]:d}) are different from other files " f"({DimensionsCheck[0]:d}, {DimensionsCheck[1]:d}). Cannot proceed") else: DimensionsCheck = myFileDimensions # check that the Projection of each layer are the same myProjection = myFile.GetProjection() if ProjectionCheck: if projectionCheck and ProjectionCheck != myProjection: raise Exception( f"Error! Projection of file {filename} {myProjection} " f"are different from other files {ProjectionCheck}. Cannot proceed") else: ProjectionCheck = myProjection # check that the GeoTransforms of each layer are the same myFileGeoTransform = myFile.GetGeoTransform(can_return_null=True) if extent == Extent.IGNORE: GeoTransformCheck = myFileGeoTransform else: Dimensions.append(myFileDimensions) GeoTransforms.append(myFileGeoTransform) if not GeoTransformCheck: GeoTransformCheck = myFileGeoTransform else: my_gt_diff = extent_util.gt_diff(GeoTransformCheck, myFileGeoTransform, eps=compatible_gt_eps, diff_support=gt_diff_support) if my_gt_diff not in [GT.SAME, GT.ALMOST_SAME]: GeoTransformDiffer = True if my_gt_diff != GT.COMPATIBLE_DIFF: raise Exception( f"Error! GeoTransform of file {filename} {myFileGeoTransform} is incompatible " f"({gt_diff_error[my_gt_diff]}), first file GeoTransform is {GeoTransformCheck}. " f"Cannot proceed") if debug: print( f"file {alpha}: {filename}, dimensions: " f"{DimensionsCheck[0]}, {DimensionsCheck[1]}, type: {myDataType[-1]}") # process allBands option allBandsIndex = None allBandsCount = 1 if allBands: if len(calc) > 1: raise Exception("Error! --allBands implies a single --calc") try: allBandsIndex = myAlphaList.index(allBands) except ValueError: raise Exception(f"Error! allBands option was given but Band {allBands} not found. Cannot proceed") allBandsCount = myFiles[allBandsIndex].RasterCount if allBandsCount <= 1: allBandsIndex = None else: allBandsCount = len(calc) if extent not in [Extent.IGNORE, Extent.FAIL] and ( GeoTransformDiffer or isinstance(extent, GeoRectangle)): # mixing different GeoTransforms/Extents GeoTransformCheck, DimensionsCheck, ExtentCheck = extent_util.calc_geotransform_and_dimensions( GeoTransforms, Dimensions, extent) if GeoTransformCheck is None: raise Exception("Error! The requested extent is empty. Cannot proceed") for i in range(len(myFileNames)): temp_vrt_filename, temp_vrt_ds = extent_util.make_temp_vrt(myFiles[i], ExtentCheck) myTempFileNames.append(temp_vrt_filename) myFiles[i] = None # close original ds myFiles[i] = temp_vrt_ds # replace original ds with vrt_ds # update the new precise dimensions and gt from the new ds GeoTransformCheck = temp_vrt_ds.GetGeoTransform() DimensionsCheck = [temp_vrt_ds.RasterXSize, temp_vrt_ds.RasterYSize] temp_vrt_ds = None ################################################################ # set up output file ################################################################ # open output file exists if outfile and os.path.isfile(outfile) and not overwrite: if allBandsIndex is not None: raise Exception("Error! allBands option was given but Output file exists, must use --overwrite option!") if len(calc) > 1: raise Exception( "Error! multiple calc options were given but Output file exists, must use --overwrite option!") if debug: print(f"Output file {outfile} exists - filling in results into file") myOut = open_ds(outfile, gdal.GA_Update) if myOut is None: error = 'but cannot be opened for update' elif [myOut.RasterXSize, myOut.RasterYSize] != DimensionsCheck: error = 'but is the wrong size' elif ProjectionCheck and ProjectionCheck != myOut.GetProjection(): error = 'but is the wrong projection' elif GeoTransformCheck and GeoTransformCheck != myOut.GetGeoTransform(can_return_null=True): error = 'but is the wrong geotransform' else: error = None if error: raise Exception( f"Error! Output exists, {error}. Use the --overwrite option " f"to automatically overwrite the existing file") myOutB = myOut.GetRasterBand(1) myOutNDV = myOutB.GetNoDataValue() myOutType = myOutB.DataType else: if outfile: # remove existing file and regenerate if os.path.isfile(outfile): os.remove(outfile) # create a new file if debug: print(f"Generating output file {outfile}") else: outfile = '' # find data type to use if not type: # use the largest type of the input files myOutType = max(myDataTypeNum) else: myOutType = type if isinstance(myOutType, str): myOutType = gdal.GetDataTypeByName(myOutType) # create file myOutDrv = gdal.GetDriverByName(format) myOut = myOutDrv.Create( os.fspath(outfile), DimensionsCheck[0], DimensionsCheck[1], allBandsCount, myOutType, creation_options) # set output geo info based on first input layer if not GeoTransformCheck: GeoTransformCheck = myFiles[0].GetGeoTransform(can_return_null=True) if GeoTransformCheck: myOut.SetGeoTransform(GeoTransformCheck) if not ProjectionCheck: ProjectionCheck = myFiles[0].GetProjection() if ProjectionCheck: myOut.SetProjection(ProjectionCheck) if NoDataValue is None: myOutNDV = None if hideNoData else DefaultNDVLookup[ myOutType] # use the default noDataValue for this datatype elif isinstance(NoDataValue, str) and NoDataValue.lower() == 'none': myOutNDV = None # not to set any noDataValue else: myOutNDV = NoDataValue # use the given noDataValue for i in range(1, allBandsCount + 1): myOutB = myOut.GetRasterBand(i) if myOutNDV is not None: myOutB.SetNoDataValue(myOutNDV) if color_table: # set color table and color interpretation if is_path_like(color_table): color_table = get_color_table(color_table) myOutB.SetRasterColorTable(color_table) myOutB.SetRasterColorInterpretation(gdal.GCI_PaletteIndex) myOutB = None # write to band myOutTypeName = gdal.GetDataTypeName(myOutType) if debug: print(f"output file: {outfile}, dimensions: {myOut.RasterXSize}, {myOut.RasterYSize}, type: {myOutTypeName}") ################################################################ # find block size to chop grids into bite-sized chunks ################################################################ # use the block size of the first layer to read efficiently myBlockSize = myFiles[0].GetRasterBand(myBands[0]).GetBlockSize() # find total x and y blocks to be read nXBlocks = (int)((DimensionsCheck[0] + myBlockSize[0] - 1) / myBlockSize[0]) nYBlocks = (int)((DimensionsCheck[1] + myBlockSize[1] - 1) / myBlockSize[1]) myBufSize = myBlockSize[0] * myBlockSize[1] if debug: print(f"using blocksize {myBlockSize[0]} x {myBlockSize[1]}") # variables for displaying progress ProgressCt = -1 ProgressMk = -1 ProgressEnd = nXBlocks * nYBlocks * allBandsCount ################################################################ # start looping through each band in allBandsCount ################################################################ for bandNo in range(1, allBandsCount + 1): ################################################################ # start looping through blocks of data ################################################################ # store these numbers in variables that may change later nXValid = myBlockSize[0] nYValid = myBlockSize[1] # loop through X-lines for X in range(0, nXBlocks): # in case the blocks don't fit perfectly # change the block size of the final piece if X == nXBlocks - 1: nXValid = DimensionsCheck[0] - X * myBlockSize[0] # find X offset myX = X * myBlockSize[0] # reset buffer size for start of Y loop nYValid = myBlockSize[1] myBufSize = nXValid * nYValid # loop through Y lines for Y in range(0, nYBlocks): ProgressCt += 1 if 10 * ProgressCt / ProgressEnd % 10 != ProgressMk and not quiet: ProgressMk = 10 * ProgressCt / ProgressEnd % 10 from sys import version_info if version_info >= (3, 0, 0): exec('print("%d.." % (10*ProgressMk), end=" ")') else: exec('print 10*ProgressMk, "..",') # change the block size of the final piece if Y == nYBlocks - 1: nYValid = DimensionsCheck[1] - Y * myBlockSize[1] myBufSize = nXValid * nYValid # find Y offset myY = Y * myBlockSize[1] # create empty buffer to mark where nodata occurs myNDVs = None # make local namespace for calculation local_namespace = {} val_lists = defaultdict(list) # fetch data for each input layer for i, Alpha in enumerate(myAlphaList): # populate lettered arrays with values if allBandsIndex is not None and allBandsIndex == i: myBandNo = bandNo else: myBandNo = myBands[i] myval = gdal_array.BandReadAsArray(myFiles[i].GetRasterBand(myBandNo), xoff=myX, yoff=myY, win_xsize=nXValid, win_ysize=nYValid) if myval is None: raise Exception(f'Input block reading failed from filename {filename[i]}') # fill in nodata values if myNDV[i] is not None: # myNDVs is a boolean buffer. # a cell equals to 1 if there is NDV in any of the corresponding cells in input raster bands. if myNDVs is None: # this is the first band that has NDV set. we initializes myNDVs to a zero buffer # as we didn't see any NDV value yet. myNDVs = numpy.zeros(myBufSize) myNDVs.shape = (nYValid, nXValid) myNDVs = 1 * numpy.logical_or(myNDVs == 1, myval == myNDV[i]) # add an array of values for this block to the eval namespace if Alpha in myAlphaFileLists: val_lists[Alpha].append(myval) else: local_namespace[Alpha] = myval myval = None for lst in myAlphaFileLists: local_namespace[lst] = val_lists[lst] # try the calculation on the array blocks this_calc = calc[bandNo - 1 if len(calc) > 1 else 0] try: myResult = eval(this_calc, global_namespace, local_namespace) except: print(f"evaluation of calculation {this_calc} failed") raise # Propagate nodata values (set nodata cells to zero # then add nodata value to these cells). if myNDVs is not None and myOutNDV is not None: myResult = ((1 * (myNDVs == 0)) * myResult) + (myOutNDV * myNDVs) elif not isinstance(myResult, numpy.ndarray): myResult = numpy.ones((nYValid, nXValid)) * myResult # write data block to the output file myOutB = myOut.GetRasterBand(bandNo) if gdal_array.BandWriteArray(myOutB, myResult, xoff=myX, yoff=myY) != 0: raise Exception('Block writing failed') myOutB = None # write to band # remove temp files for idx, tempFile in enumerate(myTempFileNames): myFiles[idx] = None os.remove(tempFile) gdal.ErrorReset() myOut.FlushCache() if gdal.GetLastErrorMsg() != '': raise Exception('Dataset writing failed') if not quiet: print("100 - Done") return myOut
def makeNDVIandEVI(state_1, rsb01, rsb02, rsb03, names, NdviPath, EviPath): areas = { 'HuangHuaiHai': [482000, 3189000, 1574000, 4662000], 'ChangJiangZhongXiaYou': [-704000, 2596000, 1574000, 3841000], 'DongBei': [1033000, 4267000, 2207000, 5922000] } outputBounds = [-704000, 2596000, 2207000, 5922000] if (state_1 is None): print("state_1 is None") return if (rsb01 is None): return if (rsb02 is None): return if (rsb03 is None): return #state1是对遥感的像元质量检测,去掉没有云的 state1 = gdal_array.BandReadAsArray(state_1.GetRasterBand(1)) b3 = gdal_array.BandReadAsArray(rsb03.GetRasterBand(1)) b2 = gdal_array.BandReadAsArray(rsb02.GetRasterBand(1)) b1 = gdal_array.BandReadAsArray(rsb01.GetRasterBand(1)) ## NDVI ndvi = ((b2 - b1) * 1.0) / ((b2 + b1) * 1.0) ndvi[numpy.isnan(ndvi)] = -9999 ndvi[numpy.isinf(ndvi)] = -9999 ndvi[ndvi > 1] = -9999 ndvi[ndvi < -1] = -9999 nodata = rsb02.GetRasterBand(1).GetNoDataValue() ndvi[b2 == nodata] = -9999 nodata = rsb01.GetRasterBand(1).GetNoDataValue() ndvi[b1 == nodata] = -9999 try: state1_10 = state1 << 10 for i in range(0, 3600): for j in range(0, 3600): if state1_10[i][j] == 0: ndvi[2 * i][2 * j] = -9999 ndvi[2 * i][2 * j + 1] = -9999 ndvi[2 * i + 1][2 * j] = -9999 ndvi[2 * i + 1][2 * j + 1] = -9999 except Exception as e: print(e) outNdvi = gdal_array.SaveArray(ndvi, "fgf", format="MEM", prototype=rsb01) NdviFileName = 'MOD09GA.%s.%s.%s.tif' % (names[1], names[3], 'ndvi') NdviFile = os.path.join(NdviPath, NdviFileName) gdal.Warp( NdviFile, outNdvi, outputBounds=outputBounds, xRes=1000, yRes=1000, srcNodata=-9999, dstNodata=-9999, outputType=gdal.GDT_Float32, dstSRS= "+proj=aea +ellps=WGS84 +datum=WGS84 +lon_0=105 +lat_1=25 +lat_2=47 +units=m +" ) #only huanghuaihai areaNdviFileName = 'MOD09GA.%s.%s.%s.tif' % (names[1], names[3], 'huanghuaihai.ndvi') areaNdviFile = os.path.join(NdviPath, areaNdviFileName) gdal.Warp(areaNdviFile, NdviFile, outputBounds=[482000, 3189000, 1574000, 4662000]) #only dongbei '''
def run_loraccs(self, ref_img_band, tgt_img_band, band_num, band_name, band_max_spectra, loess_frac, tgt_img_fp, outdir): ''' Runs the LORACCS method. ''' os.chdir(outdir) # Plot 2d histogram index = (ref_img_band>0)&(tgt_img_band>0) ref_img_band_sub = ref_img_band[index] tgt_img_band_sub = tgt_img_band[index] plt.hist2d(tgt_img_band_sub, ref_img_band_sub, bins=200, cmin = 5, cmap=plt.cm.jet, ) plt.colorbar() plt.title('%s Band 2D Histogram' %band_name) plt.xlabel('Target') plt.ylabel('Reference') save_fig = '%s_2d_hist.png' %band_name plt.savefig(save_fig) plt.show() ### Extract spectral values into a dict # Get unique values from target image tgt_uniq = np.unique(tgt_img_band) counts_dict = dict() for uniq in tgt_uniq: counts_dict[uniq] = [] img_rows = range(0, tgt_img_band.shape[0]) img_row_pixel = range(0, tgt_img_band.shape[1]) for band_row in img_rows: # iterate through rows for pixel in img_row_pixel: # iterate through pixels tgt_val = tgt_img_band[band_row][pixel] ref_val = ref_img_band[band_row][pixel] if tgt_val != 0: if ref_val != 0: # Add value to the dict values = counts_dict[tgt_val] try: values.append(ref_val) except: values = ref_val else: continue # Generate stats for uniq in tgt_uniq: values = np.array(counts_dict[uniq]) pixels = len(values) # Subset out values to get rid of outliers sub = np.sort(values) sub = sub[sub < band_max_spectra] val_sub = sub[int(len(sub) * .025) : int(len(sub) * .975)] try: mean = np.mean(val_sub) std = np.std(val_sub) except: print('Exception used') mean = np.mean(counts_dict[uniq]) std = np.std(counts_dict[uniq]) new_dict = {'values' : counts_dict[uniq], 'mean' : mean, 'std' : std, 'pixels' : pixels} counts_dict[uniq] = new_dict # Create pandas DataFrame of values spec_vals = tgt_uniq mean = [] std = [] pix = [] for uniq in tgt_uniq: mean.append(counts_dict[uniq]['mean']) std.append(counts_dict[uniq]['std']) pix.append(counts_dict[uniq]['pixels']) stats_df = pd.DataFrame() stats_df['Spec_vals'] = spec_vals stats_df['Mean'] = mean stats_df['Std'] = std stats_df['Pixels'] = pix # Remove all NaN stats_df = stats_df.fillna(0) stats_df_valid = stats_df[stats_df.Mean != 0] # Remove entries with pixel count less than 6 stats_df_valid = stats_df_valid[stats_df_valid.Pixels > 5] ### Create model # Set up params for LOESS x = stats_df_valid.Spec_vals.values y = stats_df_valid.Mean.values # Run LOESS xout, yout, wout = loess_1d(x, y, frac=loess_frac, degree=2, rotate=False) # Save values into the dataframe stats_df_valid['Mean_LOESS'] = yout # Remove any bad LOESS values (rare) stats_df_valid = stats_df_valid[stats_df_valid['Mean_LOESS'].values < band_max_spectra].copy() stats_df_valid = stats_df_valid[stats_df_valid['Mean_LOESS'].values != 0].copy() # Save the data to CSV stats_df_valid.to_csv('%s_df.csv' %band_name, index=False) # Fill gaps in spectra min_spectra = min(stats_df_valid.Spec_vals.values) max_spectra = max(stats_df_valid.Spec_vals.values) if max_spectra > band_max_spectra: reasonable_spec_vals = stats_df_valid[stats_df_valid['Spec_vals'] < band_max_spectra] max_spectra = reasonable_spec_vals['Spec_vals'].values[-1] print('Maxiumum spectral value being set to: ', max_spectra) spectral_range = range(int(min_spectra), int(max_spectra+1)) full_spectra = pd.DataFrame() full_spectra['Spec_vals'] = spectral_range full_spectra = full_spectra.merge(stats_df_valid, how='left', on='Spec_vals') full_spectra.drop(['Std'], axis=1, inplace=True) full_spectra.rename(columns={'Mean':'Org_Mean'}, inplace=True) full_spectra['Missing'] = pd.isna(full_spectra['Mean_LOESS']) # Identify missing spectral values all_y_values = [] # Predict missing spectral values for item in range(0, len(full_spectra)): if full_spectra['Missing'].iloc[item] == True: # Find nearest values on either side invalid_before_value = True n = item while invalid_before_value == True: n = n-1 invalid_before_value = full_spectra['Missing'].iloc[n] x1 = full_spectra['Spec_vals'].iloc[n] y1 = full_spectra['Mean_LOESS'].iloc[n] n = item invalid_after_value = True while invalid_after_value == True: n = n+1 invalid_after_value = full_spectra['Missing'].iloc[n] x2 = full_spectra['Spec_vals'].iloc[n] y2 = full_spectra['Mean_LOESS'].iloc[n] # Predict new spectra value using the equation of a line between points new_x = full_spectra['Spec_vals'].iloc[item] new_y = self.get_new_spec_val(x1, x2, y1, y2, new_x) else: new_y = full_spectra['Mean_LOESS'].iloc[item] all_y_values.append(new_y) full_spectra['Filled_LOESS']=all_y_values full_spectra.fillna(0, inplace=True) ### Write full spectra data frame to csv full_spectra.to_csv('%s full spectra.csv' %band_name, index=False) ### Plot result of LORACCS along with histogram fig, ax = plt.subplots(nrows=1, figsize=(6,4)) for_plot = full_spectra.copy() for_plot = for_plot[for_plot['Missing'] == False] x=for_plot['Spec_vals'].values y1=for_plot['Filled_LOESS'].values y2=for_plot['Pixels'].values y3=for_plot['Org_Mean'].values # Plot histogram ax.bar(x, y2, width=1, color='lightgray') gray_patch = mpatches.Patch(color='lightgray', label='Histogram') # Set plot to have two y axes ax2 = ax.twinx() # Original target values as a scatterplot ax2.scatter(x, y3, color='tab:gray', marker='.', label='Mean Reference') #LORACCS regression line ax2.plot(x, y1, color='tab:orange', label='LORACCS Target', linewidth=2) # Fix tick marks ylabs = ax2.get_yticks() ax2.yaxis.tick_left() ax2.set_yticklabels(ylabs, fontsize=13) ax2.yaxis.set_major_formatter(FormatStrFormatter('%.0f')) y2labs = ax.get_yticks() ax.yaxis.tick_right() ax.set_yticklabels(y2labs, fontsize=13) ax.yaxis.set_major_formatter(FormatStrFormatter('%.0f')) xlabs = ax2.get_xticks() ax2.set_xticklabels(xlabs, fontsize=13) ax2.xaxis.set_major_formatter(FormatStrFormatter('%.0f')) ax.set_title('LORACCS Model: %s Band' %band_name, fontsize=20) ax.set_xlabel('Target Spectral Values', fontsize=15) ax.yaxis.set_label_position('right') ax.set_ylabel('Reference Histogram', fontsize=15) ax2.yaxis.set_label_position('left') ax2.set_ylabel('Reference Spectral Values', fontsize=15) ax.legend(fontsize=12, loc='upper left', handles=[gray_patch]) ax2.legend(fontsize=12, loc='lower right') save_fig = '%s_LORACCS_full_spectra_plot.png' %band_name plt.savefig(save_fig) plt.show() ### Transform image using filled-in LORACCS function # Read in target image full_tgt_img = gdal.Open(tgt_img_fp) # Get bands band_data = full_tgt_img.GetRasterBand(band_num) # Read in as numpy arrays data = gdal_array.BandReadAsArray(band_data) spec_vals_dict = dict(zip(full_spectra.Spec_vals, full_spectra.Filled_LOESS)) # Change the data type in preparation for changing values data = data.astype('float') # Loop through spectral values, replace with new value / 100000. Division # necessary so already replaced values are not overwritten for spec_val in spec_vals_dict: data[data == spec_val] = spec_vals_dict[spec_val] / 100000 # Multiply by 100000 to restore proper values, return dtype data = data*100000 data = data.astype('uint16') return data # Returns band array transformed by the LORACCS method
def render_image(self, extent, size): ds = self.parent.gdal_dataset() gt = ds.GetGeoTransform() result = PIL.Image.new("RGBA", size, (0, 0, 0, 0)) # recalculate coords in pixels off_x = int((extent[0] - gt[0]) / gt[1]) off_y = int((extent[3] - gt[3]) / gt[5]) width_x = int(((extent[2] - gt[0]) / gt[1]) - off_x) width_y = int(((extent[1] - gt[3]) / gt[5]) - off_y) width_x = max(width_x, 1) width_y = max(width_y, 1) # check that pixels are not outside of image extent target_width, target_height = size offset_left = offset_top = 0 # right boundary if off_x + width_x > ds.RasterXSize: oversize_right = off_x + width_x - ds.RasterXSize target_width -= int(float(oversize_right) / width_x * target_width) width_x -= oversize_right # left boundary if off_x < 0: oversize_left = -off_x offset_left = int(float(oversize_left) / width_x * target_width) target_width -= int(float(oversize_left) / width_x * target_width) width_x -= oversize_left off_x = 0 # bottom boundary if off_y + width_y > ds.RasterYSize: oversize_bottom = off_y + width_y - ds.RasterYSize target_height -= int( float(oversize_bottom) / width_y * target_height) width_y -= oversize_bottom # top boundary if off_y < 0: oversize_top = -off_y offset_top = int(float(oversize_top) / width_y * target_height) target_height -= int(float(oversize_top) / width_y * target_height) width_y -= oversize_top off_y = 0 if target_width <= 0 or target_height <= 0: # extent doesn't intersect with image extent # return empty image return result band_count = ds.RasterCount array = numpy.zeros((target_height, target_width, band_count), numpy.uint8) for i in range(band_count): array[:, :, i] = gdal_array.BandReadAsArray(ds.GetRasterBand(i + 1), off_x, off_y, width_x, width_y, target_width, target_height) wnd = PIL.Image.fromarray(array) result.paste(wnd, (offset_left, offset_top)) return result
def get_NRMSE (self, band_list, band_names, outdir): ''' Returns a dataframe of mean-normalized RMSE values. Used to assess quality of a LORACCS-normalized image as compared to the original imagery. ''' os.chdir(outdir) reference_file = 'Ref_assessment_pixels.tif' org_file = 'Tgt_assessment_pixels.tif' loraccs_file = 'LORACCS_assessment_pixels.tif' # Set up dataframe for NRMSE values nrmse_df = pd.DataFrame(index=band_list) nrmse_df['Band Mean'] = None nrmse_df['Original NRMSE'] = None nrmse_df['LORACCS NRMSE'] = None # Get data and calculate NRMSE ref_img = gdal.Open(reference_file) org_img = gdal.Open(org_file) loraccs_img = gdal.Open(loraccs_file) for num, band in enumerate(band_list): band_num = band band_name = band_names[num] ref_data = ref_img.GetRasterBand(band) org_data = org_img.GetRasterBand(band) loraccs_data = loraccs_img.GetRasterBand(band) # Read in as numpy array ref_img_band = gdal_array.BandReadAsArray(ref_data) org_img_band = gdal_array.BandReadAsArray(org_data) loraccs_img_band = gdal_array.BandReadAsArray(loraccs_data) # Select values in array with data index = (ref_img_band>0)&(org_img_band>0) ref_img_band = np.array(ref_img_band[index]) org_img_band = np.array(org_img_band[index]) loraccs_img_band = np.array(loraccs_img_band[index]) ref_img_band.ravel() org_img_band.ravel() loraccs_img_band.ravel() # Get band mean to use for scaling band_mean = np.mean(ref_img_band) nrmse_df['Band Mean'][band] = band_mean pix_dif_array_org = abs(np.subtract((ref_img_band.astype(np.int16)), (org_img_band.astype(np.int16)))) pix_dif_array_lor = abs(np.subtract((ref_img_band.astype(np.int16)), (loraccs_img_band.astype(np.int16)))) pix_dif_org = pix_dif_array_org.ravel() pix_dif_lor = pix_dif_array_lor.ravel() pix_dif_org.sort() pix_dif_lor.sort() pix_dif_org_test = pix_dif_org[int(len(pix_dif_org) * .05) : int(len(pix_dif_org) * .95)] pix_dif_lor_test = pix_dif_lor[int(len(pix_dif_lor) * .05) : int(len(pix_dif_lor) * .95)] # Scale by band mean scaled_pix_dif_org_test = pix_dif_org_test / band_mean scaled_pix_dif_lor_test = pix_dif_lor_test / band_mean pix_org_res_sq = abs(np.square(scaled_pix_dif_org_test)) pix_lor_res_sq = abs(np.square(scaled_pix_dif_lor_test)) pix_org_res_ave = abs(np.mean(pix_org_res_sq)) pix_lor_res_ave = abs(np.mean(pix_lor_res_sq)) NRMSE_org = np.sqrt(pix_org_res_ave) NRMSE_lor = np.sqrt(pix_lor_res_ave) nrmse_df['Original NRMSE'][band] = NRMSE_org nrmse_df['LORACCS NRMSE'][band] = NRMSE_lor nrmse_df['Pixel_Cnt'] = len(pix_dif_org) nrmse_df.to_csv('NRMSE_per_band.csv') return nrmse_df
def renderTile(self, tile): bounds = tile.bounds() im = None tile_offset_left = 0 tile_offset_top = 0 if not (bounds[2] < self.data_extent[0] or bounds[0] > self.data_extent[2] or bounds[3] < self.data_extent[1] or bounds[1] > self.data_extent[3]): target_size = tile.size() off_x = int( (bounds[0] - self.geo_transform[0]) / self.geo_transform[1]) off_y = int( (bounds[3] - self.geo_transform[3]) / self.geo_transform[5]) width_x = int(( (bounds[2] - self.geo_transform[0]) / self.geo_transform[1]) - off_x) width_y = int(( (bounds[1] - self.geo_transform[3]) / self.geo_transform[5]) - off_y) # Prevent from reading off the sides of an image if off_x + width_x > self.ds.RasterXSize: oversize_right = off_x + width_x - self.ds.RasterXSize target_size = [ target_size[0] - int(float(oversize_right) / width_x * target_size[0]), target_size[1] ] width_x = self.ds.RasterXSize - off_x if off_x < 0: oversize_left = -off_x tile_offset_left = int( float(oversize_left) / width_x * target_size[0]) target_size = [ target_size[0] - int(float(oversize_left) / width_x * target_size[0]), target_size[1], ] width_x += off_x off_x = 0 if off_y + width_y > self.ds.RasterYSize: oversize_bottom = off_y + width_y - self.ds.RasterYSize target_size = [ target_size[0], target_size[1] - round(float(oversize_bottom) / width_y * target_size[1]) ] width_y = self.ds.RasterYSize - off_y if off_y < 0: oversize_top = -off_y tile_offset_top = int( float(oversize_top) / width_y * target_size[1]) target_size = [ target_size[0], target_size[1] - int(float(oversize_top) / width_y * target_size[1]), ] width_y += off_y off_y = 0 bands = self.ds.RasterCount empty_array = numpy.zeros(shape=(target_size[1], target_size[0]), dtype=numpy.uint8) rgba_array = numpy.zeros((target_size[1], target_size[0], 4), dtype=numpy.uint8) for i in range(bands): band = self.ds.GetRasterBand(i + 1) color_interp = band.GetColorInterpretation() band_array = gdalarray.BandReadAsArray(band, off_x, off_y, width_x, width_y, target_size[0], target_size[1], empty_array) if color_interp == gdal.GCI_PaletteIndex: color_table = band.GetRasterColorTable() if color_table: color_table_size = color_table.GetCount() lookup = [ numpy.arange(color_table_size), numpy.arange(color_table_size), numpy.arange(color_table_size), numpy.ones(color_table_size) * 255 ] for color_i in range(color_table_size): entry = color_table.GetColorEntry(color_i) for color_ii in range(4): lookup[color_ii][color_i] = entry[color_ii] else: lookup = [] for band_row in range(len(band_array)): for band_col in range(len(band_array[band_row])): color_pixel = numpy.zeros(shape=(4, ), dtype=numpy.uint8) for channel in range(4): color_pixel[channel] = lookup[channel][ band_array[band_row][band_col]] rgba_array[band_row, band_col] = color_pixel elif color_interp in [ gdal.GCI_RedBand, gdal.GCI_GreenBand, gdal.GCI_BlueBand, gdal.GCI_AlphaBand ]: index = { gdal.GCI_RedBand: 0, gdal.GCI_GreenBand: 1, gdal.GCI_BlueBand: 2, gdal.GCI_AlphaBand: 3, } for band_row in range(len(band_array)): for band_col in range(len(band_array[band_row])): rgba_array[band_row, band_col, 3] = 255 # TODO rgba_array[band_row, band_col, index[color_interp]] = band_array[ band_row][band_col] else: for band_row in range(len(band_array)): for band_col in range(len(band_array[band_row])): rgba_array[band_row, band_col, 0] = band_array[band_row][band_col] rgba_array[band_row, band_col, 1] = band_array[band_row][band_col] rgba_array[band_row, band_col, 2] = band_array[band_row][band_col] rgba_array[band_row, band_col, 3] = 255 im = Image.fromarray(rgba_array) big = Image.new("RGBA", tile.size(), (0, 0, 0, 0)) if im: big.paste(im, (tile_offset_left, tile_offset_top)) buf = StringIO.StringIO() big.save(buf, self.extension) buf.seek(0) tile.data = buf.read() return tile.data
def makeNDVIEVI16(satellitename, rb1, rb2, names, ndvi16path, evi16path): if (rb1 is None): return if (rb2 is None): return ''' areas = {'HuangHuaiHai': [482000, 3189000, 1574000, 4662000], 'ChangJiangZhongXiaYou': [-704000, 2596000, 1574000, 3841000], 'DongBei': [1033000, 4267000, 2207000, 5922000]}; ''' areas = { 'HuangHuaiHai': [482000, 3189000, 1574000, 4662000], 'DongBei': [1033000, 4267000, 2207000, 5922000] } ########make NDVI file########### outputBounds = [-704000, 2596000, 2207000, 5922000] NDVI16FileName = '%s.%s.%s.%s.tif' % (satellitename, names[1], names[3], "ndvi") NDVIFile = os.path.join(ndvi16path, NDVI16FileName) gdal.Warp( NDVIFile, rb1, outputBounds=outputBounds, xRes=1000, yRes=1000, srcNodata=-9999, dstNodata=-9999, outputType=gdal.GDT_Float32, dstSRS= "+proj=aea +ellps=WGS84 +datum=WGS84 +lon_0=105 +lat_1=25 +lat_2=47 +units=m +" ) for name, bound in areas.items(): areaLSTFileName = '%s.%s.%s.%s.%s.tif' % (satellitename, names[1], names[3], name, "ndvi") areaLSTFile = os.path.join(ndvi16path, areaLSTFileName) gdal.Warp(areaLSTFile, NDVIFile, outputBounds=bound) ##########make evi file################# bandevi = gdal_array.BandReadAsArray(rb2.GetRasterBand(1)) bandevi[numpy.isnan(bandevi)] = -9999 bandevi[numpy.isinf(bandevi)] = -9999 bandevi = bandevi / 10000.0 bandevi[bandevi > 2] = -9999 bandevi[bandevi < -1] = -9999 nodata = rb2.GetRasterBand(1).GetNoDataValue() bandevi[bandevi == nodata] = -9999 outevi = gdal_array.SaveArray(bandevi, "fgf", format="MEM", prototype=rb2) evi16filename = '%s.%s.%s.%s.tif' % (satellitename, names[1], names[3], "evi") eVIFile = os.path.join(evi16path, evi16filename) gdal.Warp( eVIFile, outevi, outputBounds=outputBounds, xRes=1000, yRes=1000, srcNodata=-9999, dstNodata=-9999, outputType=gdal.GDT_Float32, dstSRS= "+proj=aea +ellps=WGS84 +datum=WGS84 +lon_0=105 +lat_1=25 +lat_2=47 +units=m +" ) for name, bound in areas.items(): areaLSTFileName = '%s.%s.%s.%s.%s.tif' % (satellitename, names[1], names[3], name, "evi") areaLSTFile = os.path.join(evi16path, areaLSTFileName) gdal.Warp(areaLSTFile, eVIFile, outputBounds=bound)
def _get_map(obj, params, request): p_layers = params['LAYERS'].split(',') p_bbox = _validate_bbox([float(v) for v in params['BBOX'].split(',', 3)]) p_width = int(params['WIDTH']) p_height = int(params['HEIGHT']) p_format = params.get('FORMAT', IMAGE_FORMAT.PNG) p_style = params.get('STYLES') p_bgcolor = params.get('BGCOLOR') p_transparent = params.get('TRANSPARENT', 'FALSE') p_srs = params.get('SRS', params.get('CRS')) if p_format not in IMAGE_FORMAT.enum: raise ValidationError("Invalid FORMAT parameter.", data=dict(code="InvalidFormat")) if p_style and not (p_style == "," * (len(p_layers) - 1)): raise ValidationError("Style not found.", data=dict(code="StyleNotDefined")) if p_srs is None: raise ValidationError(message="CRS/SRS parameter required.") if p_bgcolor: r, g, b = _validate_bgcolor(p_bgcolor) bgcolor = (r, g, b) else: bgcolor = (255, 255, 255) if p_transparent.upper() == 'TRUE': img_mode = 'RGBA' bgcolor = bgcolor + (0, ) else: img_mode = 'RGB' p_size = (p_width, p_height) img = Image.new(img_mode, p_size, bgcolor) try: epsg, axis_sy = parse_srs(p_srs) except SRSParseError as e: raise ValidationError(message=str(e), data=dict(code="InvalidSRS")) try: srs = SRS.filter_by(id=epsg).one() except NoResultFound: raise ValidationError(message="SRS (id=%d) not found." % epsg, data=dict(code="InvalidSRS")) def scale(delta, img_px): dpi = 96 img_inch = float(img_px) / dpi img_m = img_inch * 0.0254 return delta / img_m xmin, ymin, xmax, ymax = p_bbox if srs.is_geographic: distance = geographic_distance(xmin, ymin, xmax, ymax) else: distance = xmax - xmin w_scale = scale(distance, p_width) for lname in p_layers: lobj = layer_by_keyname(obj, lname) res = lobj.resource request.resource_permission(DataScope.read, res) if (lobj.min_scale_denom is None or lobj.min_scale_denom >= w_scale) and \ (lobj.max_scale_denom is None or w_scale >= lobj.max_scale_denom): req = res.render_request(res.srs) # Do not use foreign SRS as it does not work correctly yet if srs.id == res.srs.id: limg = req.render_extent(p_bbox, p_size) else: mem = gdal.GetDriverByName('MEM') dst_geo = (xmin, (xmax - xmin) / p_width, 0, ymax, 0, (ymin - ymax) / p_height) dst_ds = mem.Create('', p_width, p_height, 4, gdal.GDT_Byte) dst_ds.SetGeoTransform(dst_geo) vrt = gdal.AutoCreateWarpedVRT(dst_ds, srs.wkt, res.srs.wkt) src_width = vrt.RasterXSize src_height = vrt.RasterYSize src_geo = vrt.GetGeoTransform() vrt = None src_bbox = (src_geo[0], src_geo[3] + src_geo[5] * src_height, src_geo[0] + src_geo[1] * src_width, src_geo[3]) limg = req.render_extent(src_bbox, (src_width, src_height)) if limg is not None: data = numpy.asarray(limg) img_h, img_w, band_count = data.shape src_ds = mem.Create('', src_width, src_height, band_count, gdal.GDT_Byte) src_ds.SetGeoTransform(src_geo) for i in range(band_count): bandArray = data[:, :, i] src_ds.GetRasterBand(i + 1).WriteArray(bandArray) gdal.ReprojectImage(src_ds, dst_ds, res.srs.wkt, srs.wkt) array = numpy.zeros((p_height, p_width, band_count), numpy.uint8) for i in range(band_count): array[:, :, i] = gdal_array.BandReadAsArray( dst_ds.GetRasterBand(i + 1)) limg = Image.fromarray(array) src_ds = dst_ds = None if limg is not None: img.paste(limg, (0, 0), limg) buf = BytesIO() if p_format == IMAGE_FORMAT.JPEG: img.convert('RGB').save(buf, 'jpeg') elif p_format == IMAGE_FORMAT.PNG: img.save(buf, 'png', compress_level=3) buf.seek(0) return Response(body_file=buf, content_type=p_format)
def reclassify_raster(infile, outfile, operator, threshold, pixel_value=100, frmt="GTiff", silence=True): """ Reclassify a raster into a single value according to a threshold :param infile: path to the input :param outfile: path to the output :param operator: string, possible values ">","<",">=","<=" :param threshold: threshold value :param pixel_value: the output pixel value for pixels that pass the threshold :param frmt: the output format, default "GTiff" :param silence: pass any value to enable debug info :return: None """ '''using this instead of gdal_calc because gdal calc didn't work scriptpath = os.path.join(os.path.dirname(__file__), '../', "pysdss/utility/gdal_calc.py") scriptpath = os.path.normpath(scriptpath) params = [scriptpath,"-A",path+input,"--outfile="+path+output,"--calc='(A>0.5)'", "--debug"] print("Reclassify raster") out,err = utils.run_script(params, callpy=["python3"]) print("output" + out) if err: print("ERROR:" + err) raise Exception("gdal grid failed")''' if operator not in [">", "<", ">=", "<="]: raise Exception("unknown operator") band1 = None ds1 = None bandOut = None dsOut = None try: if not silence: print("opening input") # Open the dataset ds1 = gdal.Open(infile, gdct.GA_ReadOnly) band1 = ds1.GetRasterBand(1) # Read the data into numpy arrays data1 = gdar.BandReadAsArray(band1) # get the nodata value nodata = band1.GetNoDataValue() if not silence: print("filtering input") # The actual calculation filter = "data1" + operator + str(threshold) data1[eval(filter)] = pixel_value data1[data1 != pixel_value] = nodata if not silence: print("output result") # Write the out file driver = gdal.GetDriverByName(frmt) dsOut = driver.Create(outfile, ds1.RasterXSize, ds1.RasterYSize, 1, band1.DataType) gdar.CopyDatasetInfo(ds1, dsOut) bandOut = dsOut.GetRasterBand(1) bandOut.SetNoDataValue(nodata) gdar.BandWriteArray(bandOut, data1) except RuntimeError as err: raise err except Exception as e: raise e finally: # Close the datasets if band1: band1 = None if ds1: ds1 = None if bandOut: bandOut = None if dsOut: dsOut = None