def misc_9(): old_val = gdal.GetCacheMax() gdal.SetCacheMax(3000000000) ret_val = gdal.GetCacheMax() gdal.SetCacheMax(old_val) if ret_val != 3000000000: gdaltest.post_reason('did not get expected value') print(ret_val) return 'fail' return 'success'
def mapSetup(): gdal.SetCacheMax(134217728) global emptymapstring if not exists(emptydb) and not exists(emptyosm): response = ''' { "version": 0.6, "generator": "Overpass API 0.7.55.4 3079d8ea", "osm3s": { "timestamp_osm_base": "2018-08-23T14:53:03Z", "copyright": "The data included in this document is from www.openstreetmap.org. The data is made available under ODbL." }, "elements": [] } ''' with open(emptyosm, 'w') as w: w.write(response) if not exists(emptydb): toSpatial(emptyosm, emptydb) if not exists(emptymapfile): emptymapstring = '' with open(mapfile, 'r') as f: emptymapstring = f.read() emptymapstring = emptymapstring.replace(DB_REPLACE, emptydb) else: with open(emptymapfile, 'r') as f: f.write(emptymapstring)
def mask_23(): if gdaltest.have_ng == 0: return 'skip' drv = gdal.GetDriverByName('GTiff') md = drv.GetMetadata() if md['DMD_CREATIONOPTIONLIST'].find('JPEG') == -1: return 'skip' src_ds = drv.Create('tmp/mask_23_src.tif', 3000, 2000, 3, options=['TILED=YES', 'SPARSE_OK=YES']) src_ds.CreateMaskBand(gdal.GMF_PER_DATASET) gdal.SetConfigOption('GDAL_TIFF_INTERNAL_MASK', 'YES') old_val = gdal.GetCacheMax() gdal.SetCacheMax(15000000) gdal.ErrorReset() ds = drv.CreateCopy('tmp/mask_23_dst.tif', src_ds, options=['TILED=YES', 'COMPRESS=JPEG']) gdal.SetConfigOption('GDAL_TIFF_INTERNAL_MASK', 'NO') gdal.SetCacheMax(old_val) ds = None error_msg = gdal.GetLastErrorMsg() src_ds = None drv.Delete('tmp/mask_23_src.tif') drv.Delete('tmp/mask_23_dst.tif') # 'ERROR 1: TIFFRewriteDirectory:Error fetching directory count' was triggered before if error_msg != '': return 'fail' return 'success'
def get_gdal_transformer(gcplon, gcplat, gcppixel, gcpline, xsize, ysize): """ """ gdal.SetCacheMax(2**30) gcps = [] for i, j, k, l in zip(gcplon.flat, gcplat.flat, gcppixel.flat, gcpline.flat): gcps.append(gdal.GCP(float(i), float(j), 0., float(k), float(l))) srs = osr.SpatialReference() srs.SetWellKnownGeogCS('WGS84') proj = srs.ExportToWkt() drv = gdal.GetDriverByName('MEM') dset = drv.Create('tmp', int(xsize), int(ysize)) #dset = drv.Create(id_generator(), int(xsize), int(ysize)) dset.SetGCPs(gcps, proj) #options = [''] #options = ['MAX_GCP_ORDER=3'] options = ['MAX_GCP_ORDER=-1'] transformer = gdal.Transformer(dset, None, options) return transformer
def import_geogdal(self, gdal_dataset): """ adfGeoTransform[0] /* top left x */ adfGeoTransform[1] /* w-e pixel resolution */ adfGeoTransform[2] /* 0 */ adfGeoTransform[3] /* top left y */ adfGeoTransform[4] /* 0 */ adfGeoTransform[5] /* n-s pixel resolution (negative value) */ :param gdal_dataset: a gdal dataset :return: nothing, set geooproperties from input dataset. """ gdal.SetCacheMax(2**30) self.eDT = gdal_dataset.GetRasterBand(1).DataType self.Proj = gdal_dataset.GetProjection() self.GeoTransf = gdal_dataset.GetGeoTransform() self.Driver = gdal.GetDriverByName("GTiff") self.xOrigin = self.GeoTransf[0] self.yOrigin = self.GeoTransf[3] self.pixelWidth = self.GeoTransf[1] self.pixelHeight = self.GeoTransf[5] self.srs = osr.SpatialReference() self.srs.ImportFromWkt(self.Proj) self.srsLatLon = self.srs.CloneGeogCS() self.Flag = True
def __init__(self, srcRasterfile, xoff=0, yoff=0, xsize=None, ysize=None): ''' ''' #print('Initializing reader...') self.srcRasterfile = srcRasterfile gdal.SetCacheMax(2**30) # 1 GB self.ds = gdal.Open(self.srcRasterfile, gdalconst.GA_ReadOnly) #print('self.ds: ', self.ds) if '.vrt' in self.srcRasterfile: self.fileList = self.ds.GetFileList()[1:] #print('self.fileList: ', self.fileList) self.measurement_level_ints = [] for fn in self.fileList: # default level of measurement msrlevel = conf.MSR_LEVEL_RATIO for keyword in conf.NOMINAL_KEYWORD_IN_FN: if keyword in fn: msrlevel = conf.MSR_LEVEL_NOMINAL break for key in conf.MSR_LEVELS: if conf.MSR_LEVELS[key] == msrlevel: self.measurement_level_ints.append(int(key)) break self.measurement_level_ints = np.array(self.measurement_level_ints) self.nbands = self.ds.RasterCount self.nrows = self.ds.RasterYSize self.ncols = self.ds.RasterXSize self.geotransform = self.ds.GetGeoTransform() self.projection = self.ds.GetProjection() print('%s:\n\t%d rows %d columns' % (self.srcRasterfile, self.nrows, self.ncols)) band = self.ds.GetRasterBand(1) self.nodata = band.GetNoDataValue() ## each band may have a different nodata value nodatas = [] for b in range(1, self.nbands + 1): #print('band %d nodata: %.2f' % (b, self.ds.GetRasterBand(b).GetNoDataValue())) nodatas.append(self.ds.GetRasterBand(b).GetNoDataValue()) self.nodatas = np.array(nodatas) ''' for i in range(1, self.nbands + 1): b = self.ds.GetRasterBand(1) nd = b.GetNoDataValue() print('band %d nd %.2f' % (i, nd)) ''' self.block_ysize_base = band.GetBlockSize()[0] #print('self.fileList', self.fileList) if '.vrt' in self.srcRasterfile: self.block_xsize_base = gdal.Open( self.fileList[0], gdalconst.GA_ReadOnly).GetRasterBand(1).GetBlockSize()[0] else: #self.block_xsize_base = self.ds.GetRasterBand(1).GetBlockSize()[1] self.block_xsize_base = band.GetBlockSize()[1] #print('\t%d x %d' % (self.block_xsize_base, self.block_ysize_base)) self.__N_TilesRead = 0 self.xoff, self.yoff = xoff, yoff if xsize is None: self.xsize = self.block_xsize_base elif xsize > self.ncols: print('tile xsize exceeds RasterXsize %d' % self.ncols) sys.exit(1) else: self.xsize = xsize if ysize is None: self.ysize = self.block_ysize_base elif ysize > self.nrows: print('tile xsize exceeds RasterYsize %d' % self.nrows) sys.exit(1) else: self.ysize = ysize ## estimated data size (in MB) self.estimate_TotalSize_MB = self.estimateTileSize_MB( self.nrows, self.ncols) self.estimate_TileSize_MB = self.estimateTileSize_MB( self.xsize, self.ysize) # min, max, mean, stddev self.statistics = np.zeros((self.nbands, 4)) for i in range(self.nbands): self.statistics[i] = self.ds.GetRasterBand(i + 1).GetStatistics( 0, 1) #self.statistics[i] = np.array([0, 1, 0, 1]) self.MP_pool = None
import numpy import os import re parser = argparse.ArgumentParser(description='Computes a raster file denoting which input file contained a maximum value for a given pixel.') parser.add_argument('--verbose', action='store_true') parser.add_argument('--blockSize', type=int, default=3072) parser.add_argument('--gdalCacheSize', type=int, default=2147483648) parser.add_argument('--formatOptions', nargs='*', default=['COMPRESS=DEFLATE', 'TILED=YES', 'BLOCKXSIZE=512', 'BLOCKYSIZE=512'], help='output format options') parser.add_argument('outFileName') parser.add_argument('inputFile', nargs='+') args = parser.parse_args() if args.gdalCacheSize is not None: gdal.SetCacheMax(args.gdalCacheSize) tmpSrcFile = gdal.Open(args.inputFile[0]) tmpSrcBand = tmpSrcFile.GetRasterBand(1) X = tmpSrcFile.RasterXSize Y = tmpSrcFile.RasterYSize nodataSrc = tmpSrcBand.GetNoDataValue() dataType = gdal.GDT_Byte if len(args.inputFile) < 255 else gdal.GDT_UInt16 nodataDst = 255 if len(args.inputFile) < 255 else 65535 driver = gdal.GetDriverByName('GTiff') dst = driver.Create(args.outFileName, X, Y, 1, dataType, args.formatOptions) dst.SetGeoTransform(tmpSrcFile.GetGeoTransform()) dst.SetProjection(tmpSrcFile.GetProjection()) dstBand = dst.GetRasterBand(1) dstBand.SetNoDataValue(nodataDst)
purge() t = timeit.Timer("read_raster('{0}', {1}, {2})".format( raster, x_block_size, y_block_size), setup="from __main__ import read_raster") print("\t{:.4f}s\n".format(t.timeit(1))) def purge(): subprocess.call(['/usr/bin/sudo', 'purge']) if len(sys.argv) < 2: print("Error; please pass the name of a raster file") sys.exit(1) gdal.SetCacheMax(1) raster = sys.argv[1] ds = gdal.Open(raster) band = ds.GetRasterBand(1) # Get "natural" block size, and total raster XY size. block_sizes = band.GetBlockSize() x_block_size = block_sizes[0] y_block_size = block_sizes[1] xsize = band.XSize ysize = band.YSize band = None ds = None #timer(raster, x_block_size, y_block_size * 2) #sys.exit()
def __init__(self, srcRasterfile, xoff=0, yoff=0, xsize=None, ysize=None): ''' ''' self.srcRasterfile = srcRasterfile gdal.SetCacheMax(2**30) # 1 GB self.ds = gdal.Open(self.srcRasterfile, gdalconst.GA_ReadOnly) self.fileList = self.ds.GetFileList()[1:] self.measurement_level_ints = [] for fn in self.fileList: # default level of measurement msrlevel = conf.MSR_LEVEL_RATIO for keyword in conf.NOMINAL_KEYWORD_IN_FN: if keyword in fn: msrlevel = conf.MSR_LEVEL_NOMINAL break for key in conf.MSR_LEVELS: if conf.MSR_LEVELS[key] == msrlevel: self.measurement_level_ints.append(int(key)) break self.measurement_level_ints = np.array(self.measurement_level_ints) self.nbands = self.ds.RasterCount self.nrows = self.ds.RasterYSize self.ncols = self.ds.RasterXSize self.geotransfrom = self.ds.GetGeoTransform() self.projection = self.ds.GetProjection() band = self.ds.GetRasterBand(1) self.nodata = band.GetNoDataValue() self.block_ysize_base = band.GetBlockSize()[0] self.block_xsize_base = gdal.Open( self.fileList[0], gdalconst.GA_ReadOnly).GetRasterBand(1).GetBlockSize()[0] self.__N_TilesRead = 0 self.xoff, self.yoff = xoff, yoff if xsize is None: self.xsize = self.block_xsize_base elif xsize > self.ncols: print 'tile xsize exceeds RasterXsize', self.ncols sys.exit(1) else: self.xsize = xsize if ysize is None: self.ysize = self.block_ysize_base elif ysize > self.nrows: print 'tile xsize exceeds RasterYsize', self.nrows sys.exit(1) else: self.ysize = ysize ## estimated data size (in MB) self.estimate_TotalSize_MB = self.estimateTileSize_MB( self.nrows, self.ncols) self.estimate_TileSize_MB = self.estimateTileSize_MB( self.xsize, self.ysize) self.statistics = np.zeros((self.nbands, 4)) for i in range(self.nbands): self.statistics[i] = self.ds.GetRasterBand(i + 1).GetStatistics( 0, 1) self.MP_pool = None
import gdal import glob from scipy import signal from kernels import * import numpy as np import argparse import textwrap as _textwrap from regularisation import * import logging gdal.SetCacheMax(4000000000) """ TODO * Figure out the broband conversions for the two sensors so that the kernels match up! laing paper or something? * Just noticed get negative reflectance when not enough obs for kernels... """ global ysize global xsize __author__ = "James Brennan" __copyright__ = "Copyright 2019 James Brennan" __version__ = "0.1 (06.03.2019)" __email__ = "*****@*****.**" class MultilineFormatter(argparse.HelpFormatter): def _fill_text(self, text, width, indent): text = self._whitespace_matcher.sub(' ', text).strip() paragraphs = text.split('|n ')
'Y2003_planted.tif', 'Y2004_planted.tif', 'Y2005_planted.tif', 'Y2006_planted.tif', 'Y2007_planted.tif', 'Y2008_planted.tif', 'Y2009_planted.tif', 'Y2010_planted.tif', 'Y2011_planted.tif', 'Y2012_planted.tif', 'Y2013_planted.tif', 'Y2014_planted.tif', 'Y2015_planted.tif', 'Y2016_planted.tif', 'Y2017_planted.tif', 'Y2018_planted.tif', 'Y2019_planted.tif' ] startRow = int(sys.argv[1]) endRow = startRow + int(sys.argv[2]) log("Starting") inputFiles = getInputFiles(inputDir, files) gdal.SetCacheMax(2**30) readingTime = time.time() data = readData(inputFiles, startRow, endRow) readingTime = (time.time() - readingTime) log(' Data read time: ', formatTime(readingTime), 'segs') if np.sum(data) != 0: #print('Input data:' + str(data.shape)) filterTime = time.time() dataFiltered = applyFilter(data) filterTime = (time.time() - filterTime)
def jpeg_18(): import struct height = 1024 width = 1024 src_ds = gdal.GetDriverByName('GTiff').Create('/vsimem/jpeg_18.tif', width, height, 1) for i in range(height): data = struct.pack('B' * 1, int(i / (height / 256))) src_ds.WriteRaster(0, i, width, 1, data, 1, 1) ds = gdal.GetDriverByName('JPEG').CreateCopy('/vsimem/jpeg_18.jpg', src_ds, options=['QUALITY=99']) src_ds = None gdal.Unlink('/vsimem/jpeg_18.tif') oldSize = gdal.GetCacheMax() gdal.SetCacheMax(0) line0 = ds.GetRasterBand(1).ReadRaster(0, 0, width, 1) data = struct.unpack('B' * width, line0) if abs(data[0] - 0) > 10: return 'fail' line1023 = ds.GetRasterBand(1).ReadRaster(0, height - 1, width, 1) data = struct.unpack('B' * width, line1023) if abs(data[0] - 255) > 10: return 'fail' line0_ovr1 = ds.GetRasterBand(1).GetOverview(1).ReadRaster( 0, 0, width / 4, 1) data = struct.unpack('B' * (width / 4), line0_ovr1) if abs(data[0] - 0) > 10: return 'fail' line1023_bis = ds.GetRasterBand(1).ReadRaster(0, height - 1, width, 1) if line1023_bis == line0 or line1023 != line1023_bis: gdaltest.post_reason('fail') return 'fail' line0_bis = ds.GetRasterBand(1).ReadRaster(0, 0, width, 1) if line0 != line0_bis: gdaltest.post_reason('fail') return 'fail' line255_ovr1 = ds.GetRasterBand(1).GetOverview(1).ReadRaster( 0, height / 4 - 1, width / 4, 1) data = struct.unpack('B' * (width / 4), line255_ovr1) if abs(data[0] - 255) > 10: return 'fail' line0_bis = ds.GetRasterBand(1).ReadRaster(0, 0, width, 1) if line0 != line0_bis: gdaltest.post_reason('fail') return 'fail' line0_ovr1_bis = ds.GetRasterBand(1).GetOverview(1).ReadRaster( 0, 0, width / 4, 1) if line0_ovr1 != line0_ovr1_bis: gdaltest.post_reason('fail') return 'fail' line255_ovr1_bis = ds.GetRasterBand(1).GetOverview(1).ReadRaster( 0, height / 4 - 1, width / 4, 1) if line255_ovr1 != line255_ovr1_bis: gdaltest.post_reason('fail') return 'fail' gdal.SetCacheMax(oldSize) ds = None gdal.Unlink('/vsimem/jpeg_18.jpg') return 'success'
def _create_thumbnail(red_file, green_file, blue_file, output_path, x_constraint=None, nodata=-999, work_dir=None, overwrite=True): """ Create JPEG thumbnail image using individual R, G, B images. This method comes from the old ULA codebase. :param red_file: red band data file :param green_file: green band data file :param blue_file: blue band data file :param output_path: thumbnail file to write to. :param x_constraint: thumbnail width (if not full resolution) :param nodata: null/fill data value :param work_dir: temp/work directory to use. :param overwrite: overwrite existing thumbnail? Thumbnail height is adjusted automatically to match the aspect ratio of the input images. """ nodata = int(nodata) # GDAL calls need absolute paths. thumbnail_path = pathlib.Path(output_path).absolute() if thumbnail_path.exists() and not overwrite: _LOG.warning('File already exists. Skipping creation of %s', thumbnail_path) return None, None, None # thumbnail_image = os.path.abspath(thumbnail_image) out_directory = str(thumbnail_path.parent) work_dir = os.path.abspath(work_dir) if work_dir else tempfile.mkdtemp(prefix='.thumb-tmp', dir=out_directory) try: # working files file_to = os.path.join(work_dir, 'rgb.vrt') warp_to_file = os.path.join(work_dir, 'rgb-warped.vrt') outtif = os.path.join(work_dir, 'thumbnail.tif') # Build the RGB Virtual Raster at full resolution run_command( [ "gdalbuildvrt", "-overwrite", "-separate", file_to, str(red_file), str(green_file), str(blue_file) ], work_dir ) assert os.path.exists(file_to), "VRT must exist" # Determine the pixel scaling to get the correct width thumbnail vrt = gdal.Open(file_to) intransform = vrt.GetGeoTransform() inpixelx = intransform[1] # inpixely = intransform[5] inrows = vrt.RasterYSize incols = vrt.RasterXSize # If a specific resolution is asked for. if x_constraint: outresx = inpixelx * incols / x_constraint _LOG.info('Input pixel res %r, output pixel res %r', inpixelx, outresx) outrows = int(math.ceil((float(inrows) / float(incols)) * x_constraint)) run_command([ "gdalwarp", "--config", "GDAL_CACHEMAX", str(GDAL_CACHE_MAX_MB), "-of", "VRT", "-tr", str(outresx), str(outresx), "-r", "near", "-overwrite", file_to, warp_to_file ], work_dir) else: # Otherwise use a full resolution browse image. outrows = inrows x_constraint = incols warp_to_file = file_to outresx = inpixelx _LOG.debug('Current GDAL cache max %rMB. Setting to %rMB', gdal.GetCacheMax() / 1024 / 1024, GDAL_CACHE_MAX_MB) gdal.SetCacheMax(GDAL_CACHE_MAX_MB * 1024 * 1024) # Open VRT file to array vrt = gdal.Open(warp_to_file) driver = gdal.GetDriverByName("GTiff") outdataset = driver.Create(outtif, x_constraint, outrows, 3, gdalconst.GDT_Byte) # Loop through bands and apply Scale and Offset for band_number in (1, 2, 3): band = vrt.GetRasterBand(band_number) scale, offset = _calculate_scale_offset(nodata, band) # Apply gain and offset outdataset.GetRasterBand(band_number).WriteArray( (numpy.ma.masked_less_equal(band.ReadAsArray(), nodata) * scale) + offset ) _LOG.debug('Scale %r, offset %r', scale, offset) # Must close datasets to flush to disk. # noinspection PyUnusedLocal outdataset = None # noinspection PyUnusedLocal vrt = None # GDAL Create doesn't support JPEG so we need to make a copy of the GeoTIFF run_command( [ "gdal_translate", "--config", "GDAL_CACHEMAX", str(GDAL_CACHE_MAX_MB), "-of", "JPEG", outtif, str(thumbnail_path) ], work_dir) _LOG.debug('Cleaning work files') finally: # Clean up work files if os.path.exists(work_dir): shutil.rmtree(work_dir) # Newer versions of GDAL create aux files due to the histogram. Clean them up. for f in (red_file, blue_file, green_file): f = pathlib.Path(f) aux_file = f.with_name(f.name + '.aux.xml') if aux_file.exists(): _LOG.info('Cleaning aux: %s', aux_file) os.remove(str(aux_file.absolute())) return x_constraint, outrows, outresx
def get_point_cloud(filename, progress, zone): gdal_dem_data = gdal.Open(filename) assert gdal_dem_data is not None, _('Wrong tif type.') assert gdal_dem_data.RasterCount == 1, _('Wrong tif type. Too many channels.') no_data = gdal_dem_data.GetRasterBand(1).GetNoDataValue() size = 4 * gdal_dem_data.RasterYSize * gdal_dem_data.RasterYSize step = int(np.ceil(np.sqrt(size // (512 * 1024 * 1024)))) step = max(1, step) if psutil.virtual_memory().available <= 6 * (size/step**2): raise MemoryError gdal.SetCacheMax(round((psutil.virtual_memory().available - 2 * (size/step**2)) / 2)) arr = [] for i in range(0, gdal_dem_data.RasterYSize, step): arr.append(np.copy(gdal_dem_data.ReadAsArray(0, i, gdal_dem_data.RasterXSize, 1)[:, ::step])) progress.setValue((i / gdal_dem_data.RasterYSize) * 25) QApplication.processEvents() dem = np.vstack(tuple(arr)) top_left_lon, pixel_w, turn, top_left_lat, turn_, pixel_h = gdal_dem_data.GetGeoTransform() pixel_h *= step pixel_w *= step del gdal_dem_data, arr assert top_left_lon != 0 and top_left_lat != 0, _("Couldn't define metadata (top left corner)!") progress.setValue(25) height, width = dem.shape # picture is created from top left corner x_space = (np.arange(0, width))*abs(pixel_w) + top_left_lon y_space = (np.arange(0, -height, -1))*abs(pixel_h) + top_left_lat xx, yy = np.meshgrid(x_space.astype(np.float32), y_space.astype(np.float32)) assert xx.shape == yy.shape and xx.shape == dem.shape X, Y, Z = xx.flatten(), yy.flatten(), dem.flatten() del xx, yy, dem mask = Z != no_data progress.setValue(38) QApplication.processEvents() X = X[mask] Y = Y[mask] Z = Z[mask] min_z = np.amin(Z) max_z = np.amax(Z) delta = max_z - min_z colors = np.array(((Z-min_z)/delta)*255, dtype='uint8') lut = np.empty(shape=(256, 3), dtype="uint8") # cmap = ( # (0, (255, 213, 0)), # (0.2, (255, 85, 0)), # (0.5, (255, 0, 0)), # (0.8, (230, 4, 0)), # (1.0, (192, 3, 0)) # ) cmap = ( (0, (59, 140, 64)), (0.25, (162, 166, 68)), (0.5, (214, 212, 153)), (0.75, (217, 154, 37)), (1.0, (191, 107, 33)) ) lastval, lastcol = cmap[0] for step, col in cmap[1:]: val = int(step * 256) for i in range(3): lut[lastval:val, i] = np.linspace( lastcol[i], col[i], val - lastval) lastcol = col lastval = val im_color = np.empty(shape=(colors.shape[0], 3), dtype="uint8") for i in range(3): im_color[:, i] = cv2.LUT(colors, lut[:, i]).reshape(-1) progress.setValue(84) QApplication.processEvents() X, Y, zone, letter = project_array(X, Y, zone=zone) progress.setValue(98) QApplication.processEvents() pcd = np.column_stack((X, Y, Z, im_color)) #print(pcd.shape, len(X)) #print(pcd[:, 0], pcd.T[0]) assert len(pcd.shape) == 2 and pcd.shape[0] == len(X) and pcd.shape[1] == 6 return pcd, zone, letter