def landsat7_destripe(src, outname=None):
'''
performs a gdal_fillnodata.py on src file
'''
if outname is not None:
copyfile(src, outname)
to_destripe = outname
else:
to_destripe = src
bandcount = gdal.Open(to_destripe).RasterCount
bands = [0, 1, 2, 3, 4, 6, 9]
mask = ~np.isnan(gdal.Open(to_destripe).ReadAsArray())
maskfile = './data/ex/tempmask.tif'
for i in range(1, bandcount):
export_tif(mask[i-1], gdal.Open(to_destripe), outname=maskfile,\
bands=1, dtype=gdal.GDT_Byte)
#make a copy of the file that will be destriped
src_ds = gdal.Open(to_destripe, gdalconst.GA_Update)
srcband = src_ds.GetRasterBand(i)
mask_ds = gdal.Open(maskfile)
maskband = mask_ds.GetRasterBand(1)
gdal.FillNodata(srcband,
maskband,
maxSearchDist=5,
smoothingIterations=0)
srcband = None
maskband = None
del src_ds
del mask_ds
return (print('%s made!' % outname))
def fill_nodata(inRas, maxSearchDist=5, smoothingIterations=1,
bands=[1]):
"""
fill no data using gdal
Parameters
----------
inRas: string
the input image
maxSearchDist: int
the input polygon file path
smoothingIterations: int (optional)
the clipped raster
maskBand: bool (optional)
the mask band for where to fill
"""
rds = gdal.Open(inRas, gdal.GA_Update)
# The nump and gdal dtype (ints)
# {"uint8": 1,"int8": 1,"uint16": 2,"int16": 3,"uint32": 4,"int32": 5,
# "float32": 6, "float64": 7, "complex64": 10, "complex128": 11}
# a numpy gdal conversion dict - this seems a bit long-winded
# dtypes = {"1": np.uint8, "2": np.uint16,
# "3": np.int16, "4": np.uint32,"5": np.int32,
# "6": np.float32,"7": np.float64,"10": np.complex64,
# "11": np.complex128}
# if outRas != None:
# bd = bnd = rds.GetRasterBand(bands[0])
# _copy_dataset_config(rds, FMT = 'Gtiff', outMap=outRas,
# dtype = bd.DataType, bands=rds.RasterCount)
for band in tqdm(bands):
bnd = rds.GetRasterBand(band)
# # clumsy change this
# if outRas !=None:
gdal.FillNodata(targetBand=bnd, maskBand=None,
maxSearchDist=maxSearchDist,
smoothingIterations=smoothingIterations)
rds.FlushCache()
rds=None
def gdalfill_b(b, edgemask=True, prog_func=None):
# Create mask of exterior nodata
bma = iolib.b_getma(b)
# Check to make sure there are actually holes
# if bma.count_masked() > 0:
if np.any(bma.mask):
# Create 8-bit mask_ds, and add edgemask
mask_ds = memdrv.Create('', b.XSize, b.YSize, 1, gdal.GDT_Byte)
maskband = mask_ds.GetRasterBand(1)
# The - here inverts the mask so that holes are set to 0 (invalid) and
# filled by gdalFillNodata
maskband.WriteArray((-bma.mask).astype(int))
# Now fill holes in the output
print("Filling holes")
max_distance = 40
smoothing_iterations = 0
fill_opt = []
gdal.FillNodata(b,
maskband,
max_distance,
smoothing_iterations,
fill_opt,
callback=prog_func)
# Apply the original edgemask
# Note: need to implement convexhull option like geolib.get_outline
if edgemask:
print("Generating outer edgemask")
edgemask = malib.get_edgemask(bma)
out_ma = np.ma.array(b.ReadAsArray(), mask=edgemask)
else:
out_ma = np.ma.array(b.ReadAsArray())
# Set ndv to input ndv
out_ma.set_fill_value(bma.fill_value)
# Write the filled, masked array to our tmp dataset
b.WriteArray(out_ma.filled())
# Free mask_ds
mask_ds = None
else:
print("No holes found in input band")
return b
def fill_gaps(raster_dir, output_dir, in_file, out_file, window):
print('Filling gaps...')
# fill gaps in dtm:
in_file = raster_dir.format(in_file)
out_file = output_dir.format(out_file)
copy_orig_ras = shutil.copy(in_file, out_file)
temp_file = gdal.Open(copy_orig_ras, GA_Update) #read input file
temp_band = temp_file.GetRasterBand(1) # get band 1
temp_filled = gdal.FillNodata(targetBand=temp_band,
maskBand=None,
maxSearchDist=int(window),
smoothingIterations=0)
original_ras = gdal.Open(in_file, GA_ReadOnly)
temp_filled = np.where(original_ras == -9999.0, temp_filled, original_ras)
temp_filled = None
print('...gaps filled!')
def run(self):
try:
QgsMessageLog.logMessage(
'Started task "{}"'.format(self.description()),
Lumberjack.MESSAGE_CATEGORY, Qgis.Info)
self.start_time_str = str(datetime.datetime.now())
print("=" * 30 + self.start_time_str + "=" * 30)
self.start_time = time.time()
print("DEM: ", self.dem)
print("Tree Mask: ", self.tree_mask)
# Open DEM
dataset_dem = gdal.Open(self.dem, gdal.GA_ReadOnly)
dem_raster_band = dataset_dem.GetRasterBand(1)
# Create a copy of the DEM
memory_driver = gdal.GetDriverByName('GTiff')
output_dataset = memory_driver.Create(self.output_file,
dataset_dem.RasterXSize,
dataset_dem.RasterYSize, 1,
gdal.GDT_Int16)
output_dataset.SetProjection(dataset_dem.GetProjectionRef())
output_dataset.SetGeoTransform(dataset_dem.GetGeoTransform())
outband = output_dataset.GetRasterBand(1)
outband.WriteArray(dem_raster_band.ReadAsArray())
outband.FlushCache()
dataset_dem = None
dem_raster_band = None
output_dataset = None
# Open mask
dataset_mask = gdal.Open(self.tree_mask, gdal.GA_ReadOnly)
mask_raster_band = dataset_mask.GetRasterBand(1)
array_mask = np.array(dataset_mask.ReadAsArray())
array_mask = np.array(array_mask - 1, dtype=bool)
if self.dilate_amount != 0:
array_mask = np.invert(array_mask)
array_mask = ndimage.binary_dilation(
array_mask, iterations=self.dilate_amount)
array_mask = np.invert(array_mask)
# Create mask File
memory_driver = gdal.GetDriverByName('GTiff')
output_dataset = memory_driver.Create(
self.tree_mask[:-4] + "_no_data.tif", dataset_mask.RasterXSize,
dataset_mask.RasterYSize, 1, gdal.GDT_Byte)
output_dataset.SetProjection(dataset_mask.GetProjectionRef())
output_dataset.SetGeoTransform(dataset_mask.GetGeoTransform())
outband = output_dataset.GetRasterBand(1)
outband.WriteArray(array_mask)
outband.FlushCache()
output_dataset = None
# Open copy file and mask
dataset_dem = gdal.Open(self.output_file, gdal.GA_Update)
dem_raster_band = dataset_dem.GetRasterBand(1)
dataset_mask = gdal.Open(self.tree_mask[:-4] + "_no_data.tif",
gdal.GA_ReadOnly)
mask_raster_band = dataset_mask.GetRasterBand(1)
result = gdal.FillNodata(targetBand=dem_raster_band,
maskBand=mask_raster_band,
maxSearchDist=10,
smoothingIterations=self.smooth_it)
dataset_dem = None
dataset_mask = None
self.elapsed_time = time.time() - self.start_time
print("Finished in {} seconds".format(str(self.elapsed_time)))
if self.isCanceled():
return False
return True
except Exception as e:
self.exception = e
return False
def __generate_reconstructions(self, fname):
"""
Computes wavelet decompositions and reconstructions.
:param fname: file name
"""
# need all data at once
src_ds = gdal.Open(fname, gdal.GA_ReadOnly)
od = None
if (src_ds.GetRasterBand(1).GetMaskBand() != None):
driver = gdal.GetDriverByName('GTiff')
mem_driver = gdal.GetDriverByName('MEM')
scratch = mem_driver.CreateCopy('', src_ds, strict=0)
sb = scratch.GetRasterBand(1)
nodataval = sb.GetNoDataValue()
if (nodataval is not None and self._extrapolate == False):
log.warning(' NO_DATA_VALUES found in raster %s, but not extrapolating values. This may'%(fname)+\
' cause \'ringing\' artefacts at the edges')
elif (nodataval is not None and self._extrapolate):
log.info(' Extrapolating raster %s by %d pixels' %
(fname, self._max_search_dist))
result = gdal.FillNodata(
targetBand=sb,
maskBand=None,
maxSearchDist=self._max_search_dist,
smoothingIterations=self._smoothing_iterations,
options=['TEMP_FILE_DRIVER=MEM'])
od = sb.ReadAsArray()
# GetNoDataValue() returns a double, which can be problematic when comparing against
# pixel values which may be stored as floating point values of lower precision, e.g.
# float32, float16, etc. We need to cast the 'nodatavalue' to the same format as the
# pixel values.
nodataval = getattr(np, str(od.dtype))(
sb.GetNoDataValue()) if nodataval is not None else None
# set NO_DATA_VALUE pixels to the global mean. Note that pywavelets cannot handle
# masked values
od[od == nodataval] = np.mean(od[od != nodataval])
# clean up
scratch = None
else:
od = src_ds.GetRasterBand(1).ReadAsArray()
# Compute maximum level computable for raster size
w = pywt.Wavelet(self._mother_wavelet_name)
ml = int(
np.min(
np.array([
pywt.dwt_max_level(od.shape[0], w.dec_len),
pywt.dwt_max_level(od.shape[1], w.dec_len)
])))
# generate wavelet decompositions up to required level
assert (od.ndim == 2)
#print('orig shape:', od.shape)
# reconstruct each level, starting from the highest
for l in np.arange(1, self._level + 1)[::-1]:
# Culling reconstructed levels based on highest level computable for raster size
if (l > ml):
log.warning('Maximum level computable for raster %s is %d; '
'skipping level %d' %
(os.path.basename(fname), ml, l))
continue
# Culling reconstructed levels based on user-selection
if (len(self._keep_level)):
if (l not in self._keep_level): continue
log.debug('\tReconstructing level: %d' % (l))
coeffs = pywt.wavedec2(od,
self._mother_wavelet_name,
mode=self._extension_mode,
level=l)
for i in range(1, len(coeffs)):
coeffs[i] = tuple([np.zeros_like(c) for c in coeffs[i]])
r = pywt.waverec2(coeffs,
self._mother_wavelet_name,
mode=self._extension_mode)
p = np.array(r.shape) - np.array(od.shape)
#print(p, d.shape, od.shape)
psx = pex = psy = pey = None
if (p[0] % 2):
psx = np.floor(p[0] / 2.)
pex = np.ceil(p[0] / 2.)
else:
psx = np.floor(p[0] / 2.)
pex = np.floor(p[0] / 2.)
if (p[1] % 2):
psy = np.floor(p[1] / 2.)
pey = np.ceil(p[1] / 2.)
else:
psy = np.floor(p[1] / 2.)
pey = np.floor(p[1] / 2.)
psx, pex, psy, pey = np.int_([psx, pex, psy, pey])
#print psx,pex,psy,pey
if (psx != 0 or pex != 0):
r = r[psx:-pex, :]
if (psy != 0 or pey != 0):
r = r[:, psy:-pey]
if (r.shape != od.shape):
print([r.shape, od.shape])
raise RuntimeError(
'Error encountered in wavelet reconstruction.')
fn, ext = os.path.splitext(os.path.basename(fname))
ofn = os.path.join(self._output_folder,
'%s.level_%03d%s' % (fn, l, ext))
of = driver.CreateCopy(ofn, src_ds, strict=0)
rb = of.GetRasterBand(1)
rb.WriteArray(r)
rb.ComputeStatistics(0)
of = None
# end for
src_ds = None
def main(inpix, outshape):
## inraster = "/Carnegie/DGE/caodata/Scratch/dknapp/Fish/coral_keahou_mosaic_v1k_dep_masked"
## outshape = "/Carnegie/DGE/caodata/Scratch/dknapp/Fish/coral_keahou_mosaic_v1k_dep_masked.shp"
inDS = gdal.Open(inpix, gdal.GA_ReadOnly)
mapstuff = dk.read_envi_hdr(inpix + '.hdr')
gt = (mapstuff[0], mapstuff[2], 0, mapstuff[1], 0, mapstuff[3])
## gt = inDS.GetGeoTransform()
insrs = osr.SpatialReference()
if (mapstuff[4] > 0):
insrs.ImportFromEPSG(32600 + mapstuff[4])
else:
insrs.ImportFromEPSG(32700 + abs(mapstuff[4]))
## inproj = inDS.GetProjection()
nodataval = inDS.GetRasterBand(1).GetNoDataValue()
indata = inDS.GetRasterBand(1).ReadAsArray()
if (nodataval is None):
numzero = np.equal(indata, 0.0)
if (np.sum(numzero) == 0):
# missing data value must be -9999
nodataval = -9999.0
else:
nodataval = 0.0
good = np.not_equal(indata, nodataval)
mask = np.zeros_like(indata, dtype=np.dtype('B'))
mask2 = np.zeros_like(indata, dtype=np.dtype('B'))
mask[good] = 255
mask2[good] = 1
memdrv = gdal.GetDriverByName('MEM')
tempDS = memdrv.Create('',
mask.shape[1],
mask.shape[0],
2,
eType=gdal.GDT_Byte)
tempDS.SetGeoTransform(gt)
tempDS.SetProjection(insrs.ExportToWkt())
tempDS.GetRasterBand(1).WriteArray(mask)
tempDS.GetRasterBand(2).WriteArray(mask2)
tempDS.GetRasterBand(1).SetNoDataValue(0)
tempDS.GetRasterBand(2).SetNoDataValue(0)
tempDS.FlushCache()
myBand = tempDS.GetRasterBand(1)
gdal.FillNodata(targetBand=myBand,
maskBand=None,
maxSearchDist=5,
smoothingIterations=0)
myMask = tempDS.GetRasterBand(2)
gdal.FillNodata(targetBand=myMask,
maskBand=None,
maxSearchDist=5,
smoothingIterations=0)
drv = ogr.GetDriverByName("ESRI Shapefile")
dstDS = drv.CreateDataSource(outshape)
## mysrs = osr.SpatialReference()
## mysrs.ImportFromWkt(inproj)
dst_layer = dstDS.CreateLayer("outline", srs=insrs)
newField = ogr.FieldDefn('MYFIELD', ogr.OFTInteger)
newField2 = ogr.FieldDefn('FLIGHT', ogr.OFTString)
newField2.SetWidth(80)
dst_layer.CreateField(newField)
dst_layer.CreateField(newField2)
gdal.Polygonize(myBand, myMask, dst_layer, 0, [], callback=None)
## dst_layer.FlushCache()
dst_layer.ResetReading()
print(inpix, dst_layer.GetFeatureCount())
for i in range(0, dst_layer.GetFeatureCount()):
feat = dst_layer.GetNextFeature()
dst_layer.SetFeature(feat)
feat.SetField(newField2.GetNameRef(), inpix)
dst_layer.SetFeature(feat)
dstDS.Destroy()
tempDS = None
inDS = None
import gdal from gdalconst import * mnt_file= r'D:\IVBR\DEM_filled.tif' search_dist = 1000 n_iteration = 0 mnt = gdal.Open(mnt_file, GA_Update) mnt_band = mnt.GetRasterBand(1) gdal.FillNodata(targetBand = mnt_band, maskBand=None, maxSearchDist = search_dist, smoothingIterations = n_iteration) mnt = None
