def test_basic_test_17():
from osgeo import ogr
for _ in range(2):
ogr.UseExceptions()
gdal.UseExceptions()
try:
gdal.Open('do_not_exist')
except RuntimeError:
pass
gdal.DontUseExceptions()
ogr.DontUseExceptions()
assert not gdal.GetUseExceptions()
assert not ogr.GetUseExceptions()
for _ in range(2):
ogr.UseExceptions()
gdal.UseExceptions()
try:
gdal.Open('do_not_exist')
except RuntimeError:
pass
flag = False
try:
ogr.DontUseExceptions()
gdal.DontUseExceptions()
flag = True
except:
gdal.DontUseExceptions()
ogr.DontUseExceptions()
assert not flag, 'expected failure'
assert not gdal.GetUseExceptions()
assert not ogr.GetUseExceptions()
def test_basic_test_7():
old_use_exceptions_status = gdal.GetUseExceptions()
gdal.UseExceptions()
ret = basic_test_7_internal()
if old_use_exceptions_status == 0:
gdal.DontUseExceptions()
return ret
def deleteIfExisting(filename):
"""
Delete the filename if it already exists.
If possible, use the appropriate GDAL driver to do so, to ensure
that any associated files will also be deleted.
"""
if os.path.exists(filename):
# Save the current exception-use state
usingExceptions = gdal.GetUseExceptions()
if not usingExceptions:
gdal.UseExceptions()
# Try opening it for read, to find out whether it is
# a valid GDAL file, and which driver it goes with
try:
ds = gdal.Open(str(filename))
except RuntimeError:
ds = None
if ds is not None:
# It is apparently a valid GDAL file, so get the driver appropriate for it.
drvr = ds.GetDriver()
del ds
# Use this driver to delete the file
drvr.Delete(filename)
else:
# Apparently not a valid GDAL file, for whatever reason, so just remove the file
# directly.
os.remove(filename)
# Restore exception-use state
if not usingExceptions:
gdal.DontUseExceptions()
def basic_test_17():
from osgeo import ogr
for _ in range(2):
ogr.UseExceptions()
gdal.UseExceptions()
try:
gdal.Open('do_not_exist')
except RuntimeError:
pass
gdal.DontUseExceptions()
ogr.DontUseExceptions()
if gdal.GetUseExceptions():
gdaltest.post_reason('fail')
return 'fail'
if ogr.GetUseExceptions():
gdaltest.post_reason('fail')
return 'fail'
for _ in range(2):
ogr.UseExceptions()
gdal.UseExceptions()
try:
gdal.Open('do_not_exist')
except RuntimeError:
pass
flag = False
try:
ogr.DontUseExceptions()
gdal.DontUseExceptions()
flag = True
except:
gdal.DontUseExceptions()
ogr.DontUseExceptions()
if flag:
gdaltest.post_reason('expected failure')
return 'fail'
if gdal.GetUseExceptions():
gdaltest.post_reason('fail')
return 'fail'
if ogr.GetUseExceptions():
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def vsifile_16():
old_val = gdal.GetUseExceptions()
gdal.UseExceptions()
try:
gdal.Rename('/tmp/i_dont_exist_vsifile_16.tif', '/tmp/me_neither.tif')
ret = 'fail'
except RuntimeError:
ret = 'success'
if not old_val:
gdal.DontUseExceptions()
return ret
def has_geos():
pnt1 = ogr.CreateGeometryFromWkt('POINT(10 20)')
pnt2 = ogr.CreateGeometryFromWkt('POINT(30 20)')
ogrex = ogr.GetUseExceptions()
gdalex = gdal.GetUseExceptions()
gdal.DontUseExceptions()
ogr.DontUseExceptions()
hasgeos = pnt1.Union(pnt2) is not None
if ogrex:
ogr.UseExceptions()
if gdalex:
gdal.UseExceptions()
return hasgeos
def opensAsRaster(filename):
"""
Return True if filename opens as a GDAL raster, False otherwise
"""
usingExceptions = False
if hasattr(gdal, 'GetUseExceptions'):
usingExceptions = gdal.GetUseExceptions()
gdal.UseExceptions()
try:
ds = gdal.Open(filename)
except Exception:
ds = None
opensOK = (ds is not None)
if not usingExceptions:
gdal.DontUseExceptions()
return opensOK
def open_ds(ds_path):
"""
Given a path to a raster or vector dataset, returns an opened GDAL or OGR dataset.
The caller has the responsibility of closing/deleting the dataset when finished.
:param ds_path: Path to dataset
:return: Handle to open dataset
"""
# TODO: Can be a DB Connection
# if not os.path.isfile(ds_path):
# raise Exception("Could not find file {}".format(ds_path))
# Attempt to open as gdal dataset (raster)
use_exceptions = gdal.GetUseExceptions()
gdal.UseExceptions()
logger.info("Opening the dataset: {}".format(ds_path))
try:
gdal_dataset = gdal.Open(ds_path)
if gdal_dataset:
return gdal_dataset
except RuntimeError as ex:
if ('not recognized as a supported file format' not in str(ex)) or \
('Error browsing database for PostGIS Raster tables' in str(ex)):
raise ex
finally:
if not use_exceptions:
gdal.DontUseExceptions()
# Attempt to open as ogr dataset (vector)
# ogr.UseExceptions doesn't seem to work reliably, so just check for Open returning None
ogr_dataset = ogr.Open(ds_path)
if not ogr_dataset:
logger.debug("Unknown file format: {0}".format(ds_path))
return None
return ogr_dataset
def test_basic_test_17_part_2():
# For some odd reason, this fails on the Travis CI targets after unrelated
# changes (https://travis-ci.com/github/OSGeo/gdal/jobs/501940381)
if gdaltest.skip_on_travis():
pytest.skip()
from osgeo import ogr
for _ in range(2):
ogr.UseExceptions()
gdal.UseExceptions()
flag = False
try:
ogr.DontUseExceptions()
gdal.DontUseExceptions()
flag = True
except:
gdal.DontUseExceptions()
ogr.DontUseExceptions()
assert not flag, 'expected failure'
assert not gdal.GetUseExceptions()
assert not ogr.GetUseExceptions()
def test_basic_test_11():
ds = gdal.OpenEx('data/byte.tif')
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', gdal.OF_RASTER)
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', gdal.OF_VECTOR)
assert ds is None
ds = gdal.OpenEx('data/byte.tif', gdal.OF_RASTER | gdal.OF_VECTOR)
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', gdal.OF_ALL)
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', gdal.OF_UPDATE)
assert ds is not None
ds = gdal.OpenEx(
'data/byte.tif', gdal.OF_RASTER | gdal.OF_VECTOR | gdal.OF_UPDATE
| gdal.OF_VERBOSE_ERROR)
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', allowed_drivers=[])
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', allowed_drivers=['GTiff'])
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', allowed_drivers=['PNG'])
assert ds is None
with gdaltest.error_handler():
ds = gdal.OpenEx('data/byte.tif', open_options=['FOO'])
assert ds is not None
ar_ds = [gdal.OpenEx('data/byte.tif', gdal.OF_SHARED) for _ in range(1024)]
assert ar_ds[1023] is not None
ar_ds = None
ds = gdal.OpenEx('../ogr/data/poly.shp', gdal.OF_RASTER)
assert ds is None
ds = gdal.OpenEx('../ogr/data/poly.shp', gdal.OF_VECTOR)
assert ds is not None
assert ds.GetLayerCount() == 1
assert ds.GetLayer(0) is not None
ds.GetLayer(0).GetMetadata()
ds = gdal.OpenEx('../ogr/data/poly.shp',
allowed_drivers=['ESRI Shapefile'])
assert ds is not None
ds = gdal.OpenEx('../ogr/data/poly.shp', gdal.OF_RASTER | gdal.OF_VECTOR)
assert ds is not None
ds = gdal.OpenEx('non existing')
assert ds is None and gdal.GetLastErrorMsg() == ''
gdal.PushErrorHandler('CPLQuietErrorHandler')
ds = gdal.OpenEx('non existing', gdal.OF_VERBOSE_ERROR)
gdal.PopErrorHandler()
assert ds is None and gdal.GetLastErrorMsg() != ''
old_use_exceptions_status = gdal.GetUseExceptions()
gdal.UseExceptions()
got_exception = False
try:
ds = gdal.OpenEx('non existing')
except RuntimeError:
got_exception = True
if old_use_exceptions_status == 0:
gdal.DontUseExceptions()
assert got_exception
def writeColumnToBand(gdalBand,
colName,
sequence,
colType=None,
colUsage=gdal.GFU_Generic):
"""
Given a GDAL band, Writes the data specified in sequence
(can be list, tuple or array etc)
to the named column in the attribute table assocated with the
gdalBand. colType must be one of gdal.GFT_Integer,gdal.GFT_Real,gdal.GFT_String.
can specify one of the gdal.GFU_* constants for colUsage - default is 'generic'
GDAL dataset must have been created, or opened with GA_Update
"""
if colType is None:
colType = inferColumnType(sequence)
if colType is None:
msg = "Can't infer type of column for sequence of %s" % type(
sequence[0])
raise rioserrors.AttributeTableTypeError(msg)
# check it is acually a valid type
elif colType not in (gdal.GFT_Integer, gdal.GFT_Real, gdal.GFT_String):
msg = "coltype must be a valid gdal column type"
raise rioserrors.AttributeTableTypeError(msg)
attrTbl = gdalBand.GetDefaultRAT()
if attrTbl is None:
# some formats eg ENVI return None
# here so we need to be able to cope
attrTbl = gdal.RasterAttributeTable()
isFileRAT = False
else:
isFileRAT = True
# but if it doesn't support dynamic writing
# we still ahve to call SetDefaultRAT
if not attrTbl.ChangesAreWrittenToFile():
isFileRAT = False
# We need to ensure colname doesn't already exist
colExists = False
for n in range(attrTbl.GetColumnCount()):
if attrTbl.GetNameOfCol(n) == colName:
colExists = True
colNum = n
break
if not colExists:
# preserve usage
attrTbl.CreateColumn(colName, colType, colUsage)
colNum = attrTbl.GetColumnCount() - 1
rowsToAdd = len(sequence)
# Imagine has trouble if not 256 items for byte
if gdalBand.DataType == gdal.GDT_Byte:
rowsToAdd = 256
# another hack to hide float (0-1) and int (0-255)
# color table handling.
# we assume that the column has already been created
# of the right type appropriate for the format (maybe by calcstats)
usage = attrTbl.GetUsageOfCol(colNum)
if (isColorColFromUsage(usage)
and attrTbl.GetTypeOfCol(colNum) == gdal.GFT_Real
and colType == gdal.GFT_Integer):
sequence = numpy.array(sequence, dtype=numpy.float)
sequence = sequence / 255.0
attrTbl.SetRowCount(rowsToAdd)
attrTbl.WriteArray(sequence, colNum)
if not isFileRAT:
# assume existing bands re-written
# Use GDAL's exceptions to trap the error message which arises when
# writing to a format which does not support it
usingExceptions = gdal.GetUseExceptions()
gdal.UseExceptions()
try:
gdalBand.SetDefaultRAT(attrTbl)
except Exception:
pass
if not usingExceptions:
gdal.DontUseExceptions()
def _RaiseException():
if gdal.GetUseExceptions():
_StoreLastException()
raise RuntimeError(gdal.GetLastErrorMsg())
def basic_test_11():
ds = gdal.OpenEx('data/byte.tif')
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', gdal.OF_RASTER)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', gdal.OF_VECTOR)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', gdal.OF_RASTER | gdal.OF_VECTOR)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', gdal.OF_ALL)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', gdal.OF_UPDATE)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', gdal.OF_RASTER | gdal.OF_VECTOR | gdal.OF_UPDATE | gdal.OF_VERBOSE_ERROR)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', allowed_drivers = [] )
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', allowed_drivers = ['GTiff'] )
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', allowed_drivers = ['PNG'] )
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ds = gdal.OpenEx('data/byte.tif', open_options = ['FOO'] )
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ar_ds = [ gdal.OpenEx('data/byte.tif', gdal.OF_SHARED) for i in range(1024) ]
if ar_ds[1023] is None:
gdaltest.post_reason('fail')
return 'fail'
ar_ds = None
ds = gdal.OpenEx('../ogr/data/poly.shp', gdal.OF_RASTER)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('../ogr/data/poly.shp', gdal.OF_VECTOR)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
if ds.GetLayerCount() != 1:
gdaltest.post_reason('fail')
return 'fail'
if ds.GetLayer(0) is None:
gdaltest.post_reason('fail')
return 'fail'
ds.GetLayer(0).GetMetadata()
ds = gdal.OpenEx('../ogr/data/poly.shp', allowed_drivers = ['ESRI Shapefile'] )
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('../ogr/data/poly.shp', gdal.OF_RASTER | gdal.OF_VECTOR)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('non existing')
if ds is not None or gdal.GetLastErrorMsg() != '':
gdaltest.post_reason('fail')
return 'fail'
gdal.PushErrorHandler('CPLQuietErrorHandler')
ds = gdal.OpenEx('non existing', gdal.OF_VERBOSE_ERROR)
gdal.PopErrorHandler()
if ds is not None or gdal.GetLastErrorMsg() == '':
gdaltest.post_reason('fail')
return 'fail'
old_use_exceptions_status = gdal.GetUseExceptions()
gdal.UseExceptions()
got_exception = False
try:
ds = gdal.OpenEx('non existing')
except:
got_exception = True
if old_use_exceptions_status == 0:
gdal.DontUseExceptions()
if not got_exception:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def run():
"""
Run tests of a number of more common output format drivers
"""
riostestutils.reportStart(TESTNAME)
usingExceptions = gdal.GetUseExceptions()
gdal.UseExceptions()
driverTestList = [
('HFA', ['COMPRESS=YES'], '.img'),
('GTiff', ['COMPRESS=LZW', 'TILED=YES', 'INTERLEAVE=BAND'], '.tif'),
('ENVI', ['INTERLEAVE=BSQ'], ''), ('KEA', [], '.kea')
]
# Remove any which current GDAL not suporting
driverTestList = [(drvrName, options, suffix)
for (drvrName, options, suffix) in driverTestList
if gdal.GetDriverByName(drvrName) is not None]
riostestutils.report(
TESTNAME,
'Testing drivers {}'.format(str([d[0] for d in driverTestList])))
filename = 'test.img'
riostestutils.genRampImageFile(filename)
ok = True
outfileList = []
errorList = []
for (drvr, creationOptions, suffix) in driverTestList:
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
infiles.inimg = filename
outfiles.outimg = "testout" + suffix
outfileList.append(outfiles.outimg)
controls = applier.ApplierControls()
controls.setOutputDriverName(drvr)
try:
applier.apply(copyImg, infiles, outfiles, controls=controls)
except Exception as e:
ok = False
errorList.append("{}:{}".format(drvr, str(e)))
if ok:
riostestutils.report(TESTNAME, "Passed")
else:
riostestutils.report(
TESTNAME, "Resulted in these apparent errors:\n {}".format(
'\n '.join(errorList)))
for fn in [filename] + outfileList:
if os.path.exists(fn):
drvr = gdal.Open(fn).GetDriver()
# drvr.Delete(fn)
if not usingExceptions:
gdal.DontUseExceptions()
return ok
def addStatistics(ds, progress, ignore=None, approx_ok=False):
"""
Calculates statistics and adds them to the image
Uses gdal.Band.ComputeStatistics() for mean, stddev, min and max,
and gdal.Band.GetHistogram() to do histogram calculation.
The median and mode are estimated using the histogram, and so
for larger datatypes, they will be approximate only.
For thematic layers, the histogram is calculated with as many bins
as required, for athematic integer and float types, a maximum
of 256 bins is used.
"""
progress.setLabelText("Computing Statistics...")
progress.setProgress(0)
percent = 0
percentstep = 100.0 / (ds.RasterCount * 2) # 2 steps for each layer
# flush the cache. The ensures that any unwritten data is
# written to file so we get the right stats. It also
# makes sure any metdata is written on HFA. This means
# the LAYER_TYPE setting will be picked up by rat.SetLinearBinning()
ds.FlushCache()
for bandnum in range(ds.RasterCount):
band = ds.GetRasterBand(bandnum + 1)
# fill in the metadata
tmpmeta = band.GetMetadata()
if ignore is not None:
# tell QGIS that the ignore value was ignored
band.SetNoDataValue(ignore)
tmpmeta["STATISTICS_EXCLUDEDVALUES"] = repr(
ignore) # doesn't seem to do anything
# get GDAL to calculate statistics - force recalculation. Trap errors
useExceptions = gdal.GetUseExceptions()
gdal.UseExceptions()
try:
if approx_ok and "LAYER_TYPE" in tmpmeta and tmpmeta[
"LAYER_TYPE"] == "thematic":
warnings.warn(
'WARNING: approx_ok specified for stats but image is thematic (this could be a bad idea)'
)
(minval, maxval, meanval,
stddevval) = band.ComputeStatistics(approx_ok)
except RuntimeError as e:
if str(e).endswith(
'Failed to compute statistics, no valid pixels found in sampling.'
):
minval = ignore
maxval = ignore
meanval = ignore
stddevval = 0
else:
raise e
if not useExceptions:
gdal.DontUseExceptions()
percent = percent + percentstep
progress.setProgress(percent)
tmpmeta["STATISTICS_MINIMUM"] = repr(minval)
tmpmeta["STATISTICS_MAXIMUM"] = repr(maxval)
tmpmeta["STATISTICS_MEAN"] = repr(meanval)
tmpmeta["STATISTICS_STDDEV"] = repr(stddevval)
# because we did at full res - these are the default anyway
if approx_ok:
tmpmeta["STATISTICS_APPROXIMATE"] = "YES"
else:
tmpmeta["STATISTICS_SKIPFACTORX"] = "1"
tmpmeta["STATISTICS_SKIPFACTORY"] = "1"
# create a histogram so we can do the mode and median
if band.DataType == gdal.GDT_Byte:
# if byte data use 256 bins and the whole range
histmin = 0
histmax = 255
histstep = 1.0
histCalcMin = -0.5
histCalcMax = 255.5
histnbins = 256
tmpmeta["STATISTICS_HISTOBINFUNCTION"] = 'direct'
elif "LAYER_TYPE" in tmpmeta and tmpmeta["LAYER_TYPE"] == 'thematic':
# all other thematic types a bin per value
histmin = 0
histmax = int(numpy.ceil(maxval))
histstep = 1.0
histCalcMin = -0.5
histCalcMax = maxval + 0.5
histnbins = histmax + 1
tmpmeta["STATISTICS_HISTOBINFUNCTION"] = 'direct'
elif band.DataType in gdalLargeIntTypes:
histrange = int(numpy.ceil(maxval) - numpy.floor(minval)) + 1
(histmin, histmax) = (minval, maxval)
if histrange <= 256:
histnbins = histrange
histstep = 1.0
tmpmeta["STATISTICS_HISTOBINFUNCTION"] = 'direct'
histCalcMin = histmin - 0.5
histCalcMax = histmax + 0.5
else:
histnbins = 256
tmpmeta["STATISTICS_HISTOBINFUNCTION"] = 'linear'
histCalcMin = histmin
histCalcMax = histmax
histstep = float(histCalcMax - histCalcMin) / histnbins
elif band.DataType in gdalFloatTypes:
histnbins = 256
(histmin, histmax) = (minval, maxval)
tmpmeta["STATISTICS_HISTOBINFUNCTION"] = 'linear'
histCalcMin = minval
histCalcMax = maxval
histstep = float(histCalcMax - histCalcMin) / histnbins
# Note that the complex number data types are not handled, as I am not sure
# what a histogram or a median would mean for such types.
userdata = ProgressUserData()
userdata.progress = progress
userdata.nbands = ds.RasterCount * 2
userdata.curroffset = percent
# get histogram and force GDAL to recalculate it
hist = band.GetHistogram(histCalcMin, histCalcMax, histnbins, False,
approx_ok, progressFunc, userdata)
# Check if GDAL's histogram code overflowed. This is not a fool-proof test,
# as some overflows will not result in negative counts.
histogramOverflow = (min(hist) < 0)
# we may use this ratObj reference for the colours below also
# may be None if format does not support RATs
ratObj = band.GetDefaultRAT()
if not histogramOverflow:
# comes back as a list for some reason
hist = numpy.array(hist)
# Note that we have explicitly set histstep in each datatype case
# above. In principle, this can be calculated, as it is done in the
# float case, but for some of the others we need it to be exactly
# equal to 1, so we set it explicitly there, to avoid rounding
# error problems.
# do the mode - bin with the highest count
modebin = numpy.argmax(hist)
modeval = modebin * histstep + histmin
if band.DataType == gdal.GDT_Float32 or band.DataType == gdal.GDT_Float64:
tmpmeta["STATISTICS_MODE"] = repr(modeval)
else:
tmpmeta["STATISTICS_MODE"] = repr(int(round(modeval)))
if ratObj is not None:
histIndx, histNew = findOrCreateColumn(ratObj,
gdal.GFU_PixelCount,
"Histogram",
gdal.GFT_Real)
# write the hist in a single go
ratObj.SetRowCount(histnbins)
ratObj.WriteArray(hist, histIndx)
ratObj.SetLinearBinning(histmin, (histCalcMax - histCalcMin) /
histnbins)
# The HFA driver still honours the STATISTICS_HISTOBINVALUES
# metadata item. If we are recalculating the histogram the old
# values will be copied across with the metadata so clobber it
if "STATISTICS_HISTOBINVALUES" in tmpmeta:
del tmpmeta["STATISTICS_HISTOBINVALUES"]
else:
# old method
tmpmeta["STATISTICS_HISTOBINVALUES"] = '|'.join(map(
repr, hist)) + '|'
tmpmeta["STATISTICS_HISTOMIN"] = repr(histmin)
tmpmeta["STATISTICS_HISTOMAX"] = repr(histmax)
tmpmeta["STATISTICS_HISTONUMBINS"] = repr(histnbins)
# estimate the median - bin with the middle number
middlenum = hist.sum() / 2
gtmiddle = hist.cumsum() >= middlenum
medianbin = gtmiddle.nonzero()[0][0]
medianval = medianbin * histstep + histmin
if band.DataType == gdal.GDT_Float32 or band.DataType == gdal.GDT_Float64:
tmpmeta["STATISTICS_MEDIAN"] = repr(medianval)
else:
tmpmeta["STATISTICS_MEDIAN"] = repr(int(round(medianval)))
# set the data
band.SetMetadata(tmpmeta)
if ratObj is not None and not ratObj.ChangesAreWrittenToFile():
# For drivers that require the in memory thing
band.SetDefaultRAT(ratObj)
percent = percent + percentstep
progress.setProgress(percent)
if progress.wasCancelled():
raise ProcessCancelledError()
progress.setProgress(100)
gdal.SetConfigOption('GDAL_VRT_PYTHON_TRUSTED_MODULES', ','.join(conf))
# Options declaration *************************************************************************** **
_EnvOption = namedtuple('_Option', 'sanitize, set_up, bottom_value')
_OPTIONS = {
'significant':
_EnvOption(_sanitize_significant, None, 8.0),
'default_index_dtype':
_EnvOption(_sanitize_index_dtype, None, 'int32'),
'warnings':
_EnvOption(bool, None, True),
'allow_complex_footprint':
_EnvOption(bool, None, False),
'_osgeo_use_exceptions':
_EnvOption(bool, _set_up_osgeo_use_exception, gdal.GetUseExceptions()),
'_gdal_trust_buzzard':
_EnvOption(bool, _set_up_buzz_trusted, False),
# 'check_with_invert_proj': _EnvOption(
# bool, _set_up_check_with_invert_proj,
# gdal.GetConfigOption('CHECK_WITH_INVERT_PROJ') == 'ON'
# ),
# 'default_raster_driver': _EnvOption(_sanitize_raster_driver, None, 'GTiff'),
# 'default_vector_driver': _EnvOption(_sanitize_vector_driver, None, 'ESRI Shapefile'),
# 'raster_interpolation': _EnvOption(_sanitize_raster_interpolation, None, 'area'),
}
# Storage *************************************************************************************** **
class _GlobalMapStack(object):
def resampleToReference(self,
ds,
nullValList,
workingRegion,
resamplemethod,
tempdir='.',
useVRT=False,
allowOverviewsGdalwarp=False):
"""
Resamples any inputs that do not match the reference, to the
reference image.
ds is the GDAL dataset that needs to be resampled.
nullValList is the list of null values for that dataset.
workingRegion is a PixelGridDefn
resamplemethod is a string containing a method supported by gdalwarp.
Returns the new (temporary) resampled dataset instance.
Do not call directly, use resampleAllToReference()
"""
if useVRT:
ext = '.vrt'
else:
# get the driver from the input dataset
# so we know the default extension, and type
# for the temporary file.
driver = ds.GetDriver()
drivermeta = driver.GetMetadata()
# temp image file name - based on driver extension
ext = ''
if gdal.DMD_EXTENSION in drivermeta:
ext = '.' + drivermeta[gdal.DMD_EXTENSION]
(fileh, temp_image) = tempfile.mkstemp(ext, dir=tempdir)
os.close(fileh)
overviewLevel = 'NONE'
if allowOverviewsGdalwarp:
overviewLevel = 'AUTO'
if useVRT:
driverName = 'VRT'
else:
driverName = driver.ShortName
nullValues = self.makeWarpNullOptions(nullValList)
warpOptions = gdal.WarpOptions(
overviewLevel=overviewLevel,
srcSRS=self.specialProjFixes(ds.GetProjection()),
dstSRS=self.referencePixGrid.projection,
outputBounds=[
workingRegion.xMin, workingRegion.yMin, workingRegion.xMax,
workingRegion.yMax
],
xRes=workingRegion.xRes,
yRes=workingRegion.yRes,
format=driverName,
resampleAlg=resamplemethod,
srcNodata=nullValues,
dstNodata=nullValues)
# delete later - add before the system call as if Ctrl-C is hit
# control does not always return
self.filestoremove.append(temp_image)
usingExceptions = gdal.GetUseExceptions()
gdal.UseExceptions()
try:
newds = gdal.Warp(temp_image, ds, options=warpOptions)
except Exception as e:
msg = ("Error while running gdal.Warp(): {}. Try setting the " +
"RIOS_NO_VRT_FOR_RESAMPLING environment variable " +
"to '1'").format(str(e))
raise rioserrors.GdalWarpError(msg)
finally:
if not usingExceptions:
gdal.DontUseExceptions()
# return the new dataset
return newds
def writeColumnToBand(gdalBand,
colName,
sequence,
colType=None,
colUsage=gdal.GFU_Generic):
"""
Given a GDAL band, Writes the data specified in sequence
(can be list, tuple or array etc)
to the named column in the attribute table assocated with the
gdalBand. colType must be one of gdal.GFT_Integer,gdal.GFT_Real,gdal.GFT_String.
can specify one of the gdal.GFU_* constants for colUsage - default is 'generic'
GDAL dataset must have been created, or opened with GA_Update
"""
if colType is None:
colType = inferColumnType(sequence)
if colType is None:
msg = "Can't infer type of column for sequence of %s" % type(
sequence[0])
raise rioserrors.AttributeTableTypeError(msg)
# check it is acually a valid type
elif colType not in (gdal.GFT_Integer, gdal.GFT_Real, gdal.GFT_String):
msg = "coltype must be a valid gdal column type"
raise rioserrors.AttributeTableTypeError(msg)
# things get a bit weird here as we need different
# behaviour depending on whether we have an RFC40
# RAT or not.
if hasattr(gdal.RasterAttributeTable, "WriteArray"):
# new behaviour
attrTbl = gdalBand.GetDefaultRAT()
if attrTbl is None:
# some formats eg ENVI return None
# here so we need to be able to cope
attrTbl = gdal.RasterAttributeTable()
isFileRAT = False
else:
isFileRAT = True
# but if it doesn't support dynamic writing
# we still ahve to call SetDefaultRAT
if not attrTbl.ChangesAreWrittenToFile():
isFileRAT = False
else:
# old behaviour
attrTbl = gdal.RasterAttributeTable()
isFileRAT = False
# thanks to RFC40 we need to ensure colname doesn't already exist
colExists = False
for n in range(attrTbl.GetColumnCount()):
if attrTbl.GetNameOfCol(n) == colName:
colExists = True
colNum = n
break
if not colExists:
# preserve usage
attrTbl.CreateColumn(colName, colType, colUsage)
colNum = attrTbl.GetColumnCount() - 1
rowsToAdd = len(sequence)
# Imagine has trouble if not 256 items for byte
if gdalBand.DataType == gdal.GDT_Byte:
rowsToAdd = 256
# another hack to hide float (0-1) and int (0-255)
# color table handling.
# we assume that the column has already been created
# of the right type appropriate for the format (maybe by calcstats)
# Note: this only works post RFC40 when we have an actual reference
# to the RAT rather than a new one so we can ask GetTypeOfCol
usage = attrTbl.GetUsageOfCol(colNum)
if (isColorColFromUsage(usage)
and attrTbl.GetTypeOfCol(colNum) == gdal.GFT_Real
and colType == gdal.GFT_Integer):
sequence = numpy.array(sequence, dtype=numpy.float)
sequence = sequence / 255.0
if hasattr(attrTbl, "WriteArray"):
# if GDAL > 1.10 has these functions
# thanks to RFC40
attrTbl.SetRowCount(rowsToAdd)
attrTbl.WriteArray(sequence, colNum)
elif HAVE_TURBORAT:
# use turborat to write values to RAT if available
if not isinstance(sequence, numpy.ndarray):
# turborat.writeColumn needs an array
sequence = numpy.array(sequence)
# If the dtype of the array is some unicode type, then convert to simple string type,
# as turborat does not cope with the unicode variant.
if 'U' in str(sequence.dtype):
sequence = sequence.astype(numpy.character)
turborat.writeColumn(attrTbl, colNum, sequence, rowsToAdd)
else:
defaultValues = {
gdal.GFT_Integer: 0,
gdal.GFT_Real: 0.0,
gdal.GFT_String: ''
}
# go thru and set each value into the RAT
for rowNum in range(rowsToAdd):
if rowNum >= len(sequence):
# they haven't given us enough values - fill in with default
val = defaultValues[colType]
else:
val = sequence[rowNum]
if colType == gdal.GFT_Integer:
# appears that swig cannot convert numpy.int64
# to the int type required by SetValueAsInt
# so we need to cast.
# This is a problem as readColumn returns numpy.int64
# for integer columns.
# Seems fine converting numpy.float64 to
# float however for SetValueAsDouble.
attrTbl.SetValueAsInt(rowNum, colNum, int(val))
elif colType == gdal.GFT_Real:
attrTbl.SetValueAsDouble(rowNum, colNum, float(val))
else:
attrTbl.SetValueAsString(rowNum, colNum, val)
if not isFileRAT:
# assume existing bands re-written
# Use GDAL's exceptions to trap the error message which arises when
# writing to a format which does not support it
usingExceptions = gdal.GetUseExceptions()
gdal.UseExceptions()
try:
gdalBand.SetDefaultRAT(attrTbl)
except Exception:
pass
if not usingExceptions:
gdal.DontUseExceptions()
def addStatistics(ds, progress, ignore=None):
"""
Calculates statistics and adds them to the image
Uses gdal.Band.ComputeStatistics() for mean, stddev, min and max,
and gdal.Band.GetHistogram() to do histogram calculation.
The median and mode are estimated using the histogram, and so
for larger datatypes, they will be approximate only.
For thematic layers, the histogram is calculated with as many bins
as required, for athematic integer and float types, a maximum
of 256 bins is used.
"""
progress.setLabelText("Computing Statistics...")
progress.setProgress(0)
percent = 0
percentstep = 100.0 / (ds.RasterCount * 2) # 2 steps for each layer
for bandnum in range(ds.RasterCount):
band = ds.GetRasterBand(bandnum + 1)
# fill in the metadata
tmpmeta = band.GetMetadata()
if ignore is not None:
# tell QGIS that the ignore value was ignored
band.SetNoDataValue(ignore)
tmpmeta["STATISTICS_EXCLUDEDVALUES"] = repr(
ignore) # doesn't seem to do anything
# get GDAL to calculate statistics - force recalculation. Trap errors
useExceptions = gdal.GetUseExceptions()
gdal.UseExceptions()
try:
(minval, maxval, meanval,
stddevval) = band.ComputeStatistics(False)
except RuntimeError as e:
if str(e).endswith(
'Failed to compute statistics, no valid pixels found in sampling.'
):
minval = ignore
maxval = ignore
meanval = ignore
stddevval = 0
if not useExceptions:
gdal.DontUseExceptions()
percent = percent + percentstep
progress.setProgress(percent)
tmpmeta["STATISTICS_MINIMUM"] = repr(minval)
tmpmeta["STATISTICS_MAXIMUM"] = repr(maxval)
tmpmeta["STATISTICS_MEAN"] = repr(meanval)
tmpmeta["STATISTICS_STDDEV"] = repr(stddevval)
# because we did at full res - these are the default anyway
tmpmeta["STATISTICS_SKIPFACTORX"] = "1"
tmpmeta["STATISTICS_SKIPFACTORY"] = "1"
# create a histogram so we can do the mode and median
if band.DataType == gdal.GDT_Byte:
# if byte data use 256 bins and the whole range
histmin = 0
histmax = 255
histstep = 1.0
histCalcMin = -0.5
histCalcMax = 255.5
histnbins = 256
tmpmeta["STATISTICS_HISTOBINFUNCTION"] = 'direct'
elif "LAYER_TYPE" in tmpmeta and tmpmeta["LAYER_TYPE"] == 'thematic':
# all other thematic types a bin per value
histmin = 0
histmax = int(numpy.ceil(maxval))
histstep = 1.0
histCalcMin = -0.5
histCalcMax = maxval + 0.5
histnbins = histmax + 1
tmpmeta["STATISTICS_HISTOBINFUNCTION"] = 'direct'
elif band.DataType in gdalLargeIntTypes:
histrange = int(numpy.ceil(maxval) - numpy.floor(minval)) + 1
(histmin, histmax) = (minval, maxval)
if histrange <= 256:
histnbins = histrange
histstep = 1.0
tmpmeta["STATISTICS_HISTOBINFUNCTION"] = 'direct'
histCalcMin = histmin - 0.5
histCalcMax = histmax + 0.5
else:
histnbins = 256
tmpmeta["STATISTICS_HISTOBINFUNCTION"] = 'linear'
histCalcMin = histmin
histCalcMax = histmax
histstep = float(histCalcMax - histCalcMin) / histnbins
elif band.DataType in gdalFloatTypes:
histnbins = 256
(histmin, histmax) = (minval, maxval)
tmpmeta["STATISTICS_HISTOBINFUNCTION"] = 'linear'
histCalcMin = minval
histCalcMax = maxval
histstep = float(histCalcMax - histCalcMin) / histnbins
# Note that the complex number data types are not handled, as I am not sure
# what a histogram or a median would mean for such types.
userdata = ProgressUserData()
userdata.progress = progress
userdata.nbands = ds.RasterCount * 2
userdata.curroffset = percent
# get histogram and force GDAL to recalculate it
hist = band.GetHistogram(histCalcMin, histCalcMax, histnbins, False,
False, progressFunc, userdata)
# Check if GDAL's histogram code overflowed. This is not a fool-proof test,
# as some overflows will not result in negative counts.
histogramOverflow = (min(hist) < 0)
# we may use this rat reference for the colours below also
# may be None if format does not support RATs
rat = band.GetDefaultRAT()
if not histogramOverflow:
# comes back as a list for some reason
hist = numpy.array(hist)
# Note that we have explicitly set histstep in each datatype case
# above. In principle, this can be calculated, as it is done in the
# float case, but for some of the others we need it to be exactly
# equal to 1, so we set it explicitly there, to avoid rounding
# error problems.
# do the mode - bin with the highest count
modebin = numpy.argmax(hist)
modeval = modebin * histstep + histmin
if band.DataType == gdal.GDT_Float32 or band.DataType == gdal.GDT_Float64:
tmpmeta["STATISTICS_MODE"] = repr(modeval)
else:
tmpmeta["STATISTICS_MODE"] = repr(int(round(modeval)))
tmpmeta["STATISTICS_HISTOMIN"] = repr(histmin)
tmpmeta["STATISTICS_HISTOMAX"] = repr(histmax)
tmpmeta["STATISTICS_HISTONUMBINS"] = repr(histnbins)
if haveRFC40 and rat is not None:
histIndx, histNew = findOrCreateColumn(rat,
gdal.GFU_PixelCount,
"Histogram",
gdal.GFT_Real)
# write the hist in a single go
rat.SetRowCount(histnbins)
rat.WriteArray(hist, histIndx)
# The HFA driver still honours the STATISTICS_HISTOBINVALUES
# metadata item. If we are recalculating the histogram the old
# values will be copied across with the metadata so clobber it
if "STATISTICS_HISTOBINVALUES" in tmpmeta:
del tmpmeta["STATISTICS_HISTOBINVALUES"]
else:
# old method
tmpmeta["STATISTICS_HISTOBINVALUES"] = '|'.join(map(
repr, hist)) + '|'
# estimate the median - bin with the middle number
middlenum = hist.sum() / 2
gtmiddle = hist.cumsum() >= middlenum
medianbin = gtmiddle.nonzero()[0][0]
medianval = medianbin * histstep + histmin
if band.DataType == gdal.GDT_Float32 or band.DataType == gdal.GDT_Float64:
tmpmeta["STATISTICS_MEDIAN"] = repr(medianval)
else:
tmpmeta["STATISTICS_MEDIAN"] = repr(int(round(medianval)))
# set the data
band.SetMetadata(tmpmeta)
# if it is thematic and there is no colour table
# add one because Imagine fails in weird ways otherwise
# we make a random colour table to make it obvious
if "LAYER_TYPE" in tmpmeta and tmpmeta["LAYER_TYPE"] == 'thematic':
# old way
if (not haveRFC40 or rat is None) and band.GetColorTable() is None:
import random # this also seeds on the time
colorTable = gdal.ColorTable()
alpha = 255
for i in range(histnbins):
c1 = int(random.random() * 255)
c2 = int(random.random() * 255)
c3 = int(random.random() * 255)
entry = (c1, c2, c3, alpha)
colorTable.SetColorEntry(i, entry)
#band.SetColorTable(colorTable)
elif haveRFC40 and rat is not None:
# check all the columns
redIdx, redNew = findOrCreateColumn(rat, gdal.GFU_Red, "Red",
gdal.GFT_Integer)
greenIdx, greenNew = findOrCreateColumn(
rat, gdal.GFU_Green, "Green", gdal.GFT_Integer)
blueIdx, blueNew = findOrCreateColumn(rat, gdal.GFU_Blue,
"Blue", gdal.GFT_Integer)
alphaIdx, alphaNew = findOrCreateColumn(
rat, gdal.GFU_Alpha, "Alpha", gdal.GFT_Integer)
# were any of these not already existing?
if redNew or greenNew or blueNew or alphaNew:
data = numpy.random.randint(0, 256, histnbins)
rat.WriteArray(data, redIdx)
data = numpy.random.randint(0, 256, histnbins)
rat.WriteArray(data, greenIdx)
data = numpy.random.randint(0, 256, histnbins)
rat.WriteArray(data, blueIdx)
data = numpy.empty(histnbins, dtype=numpy.int)
data.fill(255)
#rat.WriteArray(data, alphaIdx)
if haveRFC40 and rat is not None and not rat.ChangesAreWrittenToFile():
# For drivers that require the in memory thing
band.SetDefaultRAT(rat)
percent = percent + percentstep
progress.setProgress(percent)
if progress.wasCancelled():
raise ProcessCancelledError()
progress.setProgress(100)