def read_grid(RTG_file, rti, RTG_type='FLOAT', REPORT=False, SILENT=True):
#----------------------------------------------------------
# Notes: This function is outside of the "rtg_file" class
# and will be more convenient in many cases.
#----------------------------------------------------------
if not (SILENT):
print 'Reading grid values...'
#-------------------
# Read in the grid
#-------------------
file_unit = open(RTG_file, 'rb')
dtype = rti_files.get_numpy_data_type(RTG_type)
grid = numpy.fromfile(file_unit, count=rti.n_pixels, dtype=dtype)
grid = grid.reshape(rti.nrows, rti.ncols)
if (rti.SWAP_ENDIAN):
grid.byteswap(True)
file_unit.close()
if not (SILENT):
print 'Finished reading grid from:'
print ' ' + RTG_file
#------------------
# Optional report
#------------------
if (REPORT):
print ' min(grid) =', grid.min()
print ' max(grid) =', grid.max()
print ' '
return grid
def read_grid(self,
dtype='float32',
rtg_type=None,
REPORT=False,
VERBOSE=False):
if (VERBOSE):
print 'Reading grid values....'
#--------------------------------------
# Different ways to specify data type
#--------------------------------------
if (rtg_type != None):
dtype = rti_files.get_numpy_data_type(rtg_type)
#--------------------------------
# Read grid as binary from file
#--------------------------------
n_values = self.nx * self.ny
grid = numpy.fromfile(self.rtg_unit, count=n_values, dtype=dtype)
grid = grid.reshape(self.ny, self.nx)
#----------------
# Close the file
#----------------
self.rtg_unit.close()
#--------------------------------
# Swap byte order, if necessary
#--------------------------------
if (self.info.SWAP_ENDIAN): grid.byteswap(True)
#------------------
# Optional report
#------------------
if (REPORT):
gmin = grid.min()
gmax = grid.max()
print ' min(grid) =', gmin
print ' max(grid) =', gmax
if (VERBOSE):
print 'Finished reading grid from:'
print ' ' + self.file_name
return grid
def read_grid(self, dtype='float32', rtg_type=None,
REPORT=False, VERBOSE=False):
if (VERBOSE):
print 'Reading grid values....'
#--------------------------------------
# Different ways to specify data type
#--------------------------------------
if (rtg_type != None):
dtype = rti_files.get_numpy_data_type( rtg_type )
#--------------------------------
# Read grid as binary from file
#--------------------------------
n_values = self.nx * self.ny
grid = numpy.fromfile( self.rtg_unit, count=n_values,
dtype=dtype )
grid = grid.reshape( self.ny, self.nx )
#----------------
# Close the file
#----------------
self.rtg_unit.close()
#--------------------------------
# Swap byte order, if necessary
#--------------------------------
if (self.info.SWAP_ENDIAN): grid.byteswap( True )
#------------------
# Optional report
#------------------
if (REPORT):
gmin = grid.min()
gmax = grid.max()
print ' min(grid) =', gmin
print ' max(grid) =', gmax
if (VERBOSE):
print 'Finished reading grid from:'
print ' ' + self.file_name
return grid
def read_grid( RTG_file, rti, RTG_type='FLOAT',
REPORT=False, SILENT=True ):
#----------------------------------------------------------
# Notes: This function is outside of the "rtg_file" class
# and will be more convenient in many cases.
#----------------------------------------------------------
if not(SILENT):
print 'Reading grid values...'
#-------------------
# Read in the grid
#-------------------
file_unit = open( RTG_file, 'rb' )
dtype = rti_files.get_numpy_data_type( RTG_type )
grid = numpy.fromfile( file_unit, count=rti.n_pixels,
dtype=dtype )
grid = grid.reshape( rti.nrows, rti.ncols )
if (rti.SWAP_ENDIAN):
grid.byteswap( True )
file_unit.close()
if not(SILENT):
print 'Finished reading grid from:'
print ' ' + RTG_file
#------------------
# Optional report
#------------------
if (REPORT):
print ' min(grid) =', grid.min()
print ' max(grid) =', grid.max()
print ' '
return grid
def fill_pits(DEM, DEM_type, nx, ny,
nodata=float32(-9999),
USE_64_BITS=False, SILENT=True):
#----------------------------------------------------------
# Notes: This RAM-based version is about 4 times faster
# than a similar file-based version.
#----------------------------------------------------------
# Notes: Assume that original DEM has already been
# copied to raw_DEM_file.
# Be sure to call Check_Overwrite on files.
#----------------------------------------------------------
# DEM Time (s) Max heap Same? New values
#----------------------------------------------------------
# Small_KY 2925 Yes 419
# KY_Sub 9.59 21280 Yes 4901
# Beaver 65.86 83212 Yes 52353
# Jing
# Loess2 781582 Yes 1543427
#----------------------------------------------------------
# Small (132 x 108) = 14256
# KY_Sub (408 x 468) = 190944 (5160.6 pix/sec)
# Beaver (915 x 1405) = 1285575 (4652.7 pix/sec)
# Jing (3312 x 3771) = 12489552
# Loess2 (6016 x 4250) = 25568000 (4674.3 pix/sec)
# Amazon (37558 x 25009) = 939288022 (2.3 days ??)
# S.Amer (56871 x 84906) = 4828689126 (11.89 days ??)
#----------------------------------------------------------
USE_MY_HEAP = False
### USE_MY_HEAP = True
start = time.time()
if (USE_64_BITS):
ID_type = 'Int64'
nx = numpy.int64(nx)
ny = numpy.int64(ny)
else:
ID_type = 'Int32'
nx = numpy.int32(nx)
ny = numpy.int32(ny)
n_pixels = (nx * ny)
#--------------------------------------------
# Get the "start pixels" which include:
# (1) all edge pixels with valid data
# (2) all interior pixels with valid data
# beside a pixel with a nodata value
# (3) all pixels with "closed basin" code
#--------------------------------------------
# Do we need to remove duplicate entries ?
#--------------------------------------------
#*** IDs = get_start_pixels(DEM, nx, ny, /EDGES_ONLY)
IDs = get_start_pixels(DEM, nx, ny)
n_IDs = size(IDs)
#------------------------------
# Initialize the CLOSED array
#----------------------------------------
# Set all nodata & NaN pixels to CLOSED
#----------------------------------------
OPEN = uint8(0)
CLOSED = uint8(1)
ON_HEAP = uint8(2)
## C = numpy.zeros( (ny, nx), dtype='UInt8' ).flat # (iterator for 1D indices)
C = numpy.zeros( n_pixels, dtype='UInt8' )
w = where( logical_or((DEM <= nodata), (isfinite(DEM) == 0)) )
nw = size(w[0])
w = w[0] ############## (need this ?)
if (nw != 0):
C[w] = CLOSED
n_closed = int64( nw )
n_raised = int64(0)
if not(SILENT):
print 'Number of nodata and NaN values =', nw
print 'Finished initializing "closed" array.'
print ' '
#------------------------------------------
# Initialize variables for priority queue
# including arrays called heap and IDs
#------------------------------------------
if not(SILENT):
print 'Putting boundary pixels on heap...'
if (USE_MY_HEAP):
#-------------------------------------
# Use methods of my own "heap" class
#-------------------------------------
DEM_dtype = rti_files.get_numpy_data_type( DEM_type )
heap = heap_base.heap_base()
heap.initialize(nx, ny, DEM_dtype)
#----------------------------------------
# Add the "boundary pixels" to the heap
#----------------------------------------
### for k in xrange(n_IDs): # (do a speed comparison later)
for ID in IDs:
#-------------------------
# Read pixel's elevation
#-------------------------
## ID = IDs[k] # (for speed comparison)
zval = DEM.flat[ ID ]
if (zval > nodata) and (isfinite(zval) == 1):
heap.insert( zval, ID )
n_heap = heap.n
else:
#------------------------------------------
# Use the built-in Python package "heapq"
# which should be a lot faster (in C).
#----------------------------------------------
# numpy.column_stack() is similar to Python's
# built-in "zip()", but should be faster
#----------------------------------------------
## fast_heap = numpy.column_stack( (DEM.flat[ IDs ], IDs) )
## fast_heap = fast_heap.tolist()
fast_heap = zip( DEM.flat[ IDs ], IDs )
heapq.heapify( fast_heap )
n_heap = len( fast_heap )
#--------------------------------------------
# Should this be here ? (Added on 11/03/09)
# Should make code a little faster ?
#--------------------------------------------
C[ IDs ] = ON_HEAP
if not(SILENT):
print 'Number of pixels on heap = ' + str(n_heap)
print 'Finished with heap insertion.'
print ' '
#----------------------------------
# Prepare to compute neighbor IDs
#----------------------------------
incs = array([-nx - 1, -nx, -nx + 1, -1, 1, nx - 1, nx, nx + 1], dtype=ID_type)
tstr = str(n_pixels)
#------------------------------------------
# Process pixels to fill pits until there
# are no pixels left on the heap
#------------------------------------------
# NB! Both while loops work for Small,
# KY_Sub, but not Beaver
#-----------------------------------------
while (n_heap > 0):
### while (n_closed < n_pixels):
#--------------------------------------
# Print status message every so often
#--------------------------------------
if ((n_closed % 5000) == 0):
nstr = str(n_closed) + ' of '
if not(SILENT):
print 'n_closed = ' + nstr + tstr
time.sleep(float32(0.001))
#----------------------------------------
# Get pixel in heap with min elevation,
# and remove it from the heap array
#----------------------------------------
if (USE_MY_HEAP):
min_ID = heap.get_min_ID()
zmin = heap.get_min()
else:
pair = heapq.heappop( fast_heap )
min_ID = pair[1]
zmin = pair[0]
## print 'zmin =', zmin
#------------------------------------------
# Flag pixel with min elevation as CLOSED
#------------------------------------------
C[ min_ID ] = CLOSED
n_closed += 1
#------------------------------------
# Get IDs of this pixel's neighbors
#------------------------------------
ID_vals = (min_ID + incs)
#-------------------------------------------
# Find neighbors with valid IDs that are
# still OPEN (i.e. not(ON_HEAP or CLOSED))
#-------------------------------------------
w = where( logical_and((ID_vals >= 0), (ID_vals < n_pixels)) )
if (size(w[0]) != 0):
ID_vals = ID_vals[ w[0] ]
#-------------------------------------
# Next line excludes neighbor pixels
# that are either ON_HEAP or CLOSED
#-------------------------------------
C_vals = C[ ID_vals ]
w2 = where( C_vals == OPEN )
if (size(w2[0]) != 0):
ID_vals = ID_vals[ w2[0] ]
else:
ID_vals = []
#--------------------------------------
# Process each of the neighbor pixels
#--------------------------------------
for ID in ID_vals:
#------------------------------------
# Raise neighbor elevation,
# if it is smaller than zmin & OPEN
#------------------------------------
if (DEM.flat[ ID ] < zmin):
DEM.flat[ ID] = zmin
n_raised += 1
## DEM.flat[ ID ] = numpy.maximum( DEM.flat[ ID ], zmin )
#---------------------------
# Flag neighbor as ON_HEAP
#---------------------------
C[ ID ] = ON_HEAP
#---------------------------------
# Add neighbor to priority queue
#---------------------------------
if (USE_MY_HEAP):
heap.insert( DEM.flat[ ID ], ID )
else:
heapq.heappush( fast_heap, (DEM.flat[ ID ], ID) )
if (USE_MY_HEAP):
n_heap = heap.n
else:
n_heap = len( fast_heap )
#---------------------
# Print final report
#---------------------
if not(SILENT):
print 'Total pixels = ' + tstr
print 'Raised pixels = ' + str(n_raised)
print 'Drained pixels = ' + str(n_closed)
if (USE_MY_HEAP):
print 'Max heap size = ' + str(heap.nmax)
print ' '
#-----------------------------------------------------
# Would be better to use CSDMS_base.print_run_time()
#-----------------------------------------------------
run_time = (time.time() - start)
rt_str = ('%10.4f' % run_time) + ' [seconds]'
print 'Run time for fill_pits() = ' + rt_str
print 'Finished with fill_pits().'
print ' '
def fill_pits(DEM,
DEM_type,
nx,
ny,
nodata=float32(-9999),
USE_64_BITS=False,
SILENT=True):
#----------------------------------------------------------
# Notes: This RAM-based version is about 4 times faster
# than a similar file-based version.
#----------------------------------------------------------
# Notes: Assume that original DEM has already been
# copied to raw_DEM_file.
# Be sure to call Check_Overwrite on files.
#----------------------------------------------------------
# DEM Time (s) Max heap Same? New values
#----------------------------------------------------------
# Small_KY 2925 Yes 419
# KY_Sub 9.59 21280 Yes 4901
# Beaver 65.86 83212 Yes 52353
# Jing
# Loess2 781582 Yes 1543427
#----------------------------------------------------------
# Small (132 x 108) = 14256
# KY_Sub (408 x 468) = 190944 (5160.6 pix/sec)
# Beaver (915 x 1405) = 1285575 (4652.7 pix/sec)
# Jing (3312 x 3771) = 12489552
# Loess2 (6016 x 4250) = 25568000 (4674.3 pix/sec)
# Amazon (37558 x 25009) = 939288022 (2.3 days ??)
# S.Amer (56871 x 84906) = 4828689126 (11.89 days ??)
#----------------------------------------------------------
USE_MY_HEAP = False
### USE_MY_HEAP = True
start = time.time()
if (USE_64_BITS):
ID_type = 'Int64'
nx = numpy.int64(nx)
ny = numpy.int64(ny)
else:
ID_type = 'Int32'
nx = numpy.int32(nx)
ny = numpy.int32(ny)
n_pixels = (nx * ny)
#--------------------------------------------
# Get the "start pixels" which include:
# (1) all edge pixels with valid data
# (2) all interior pixels with valid data
# beside a pixel with a nodata value
# (3) all pixels with "closed basin" code
#--------------------------------------------
# Do we need to remove duplicate entries ?
#--------------------------------------------
#*** IDs = get_start_pixels(DEM, nx, ny, /EDGES_ONLY)
IDs = get_start_pixels(DEM, nx, ny)
n_IDs = size(IDs)
#------------------------------
# Initialize the CLOSED array
#----------------------------------------
# Set all nodata & NaN pixels to CLOSED
#----------------------------------------
OPEN = uint8(0)
CLOSED = uint8(1)
ON_HEAP = uint8(2)
## C = numpy.zeros( (ny, nx), dtype='UInt8' ).flat # (iterator for 1D indices)
C = numpy.zeros(n_pixels, dtype='UInt8')
w = where(logical_or((DEM <= nodata), (isfinite(DEM) == 0)))
nw = size(w[0])
w = w[0] ############## (need this ?)
if (nw != 0):
C[w] = CLOSED
n_closed = int64(nw)
n_raised = int64(0)
if not (SILENT):
print 'Number of nodata and NaN values =', nw
print 'Finished initializing "closed" array.'
print ' '
#------------------------------------------
# Initialize variables for priority queue
# including arrays called heap and IDs
#------------------------------------------
if not (SILENT):
print 'Putting boundary pixels on heap...'
if (USE_MY_HEAP):
#-------------------------------------
# Use methods of my own "heap" class
#-------------------------------------
DEM_dtype = rti_files.get_numpy_data_type(DEM_type)
heap = heap_base.heap_base()
heap.initialize(nx, ny, DEM_dtype)
#----------------------------------------
# Add the "boundary pixels" to the heap
#----------------------------------------
### for k in xrange(n_IDs): # (do a speed comparison later)
for ID in IDs:
#-------------------------
# Read pixel's elevation
#-------------------------
## ID = IDs[k] # (for speed comparison)
zval = DEM.flat[ID]
if (zval > nodata) and (isfinite(zval) == 1):
heap.insert(zval, ID)
n_heap = heap.n
else:
#------------------------------------------
# Use the built-in Python package "heapq"
# which should be a lot faster (in C).
#----------------------------------------------
# numpy.column_stack() is similar to Python's
# built-in "zip()", but should be faster
#----------------------------------------------
## fast_heap = numpy.column_stack( (DEM.flat[ IDs ], IDs) )
## fast_heap = fast_heap.tolist()
fast_heap = zip(DEM.flat[IDs], IDs)
heapq.heapify(fast_heap)
n_heap = len(fast_heap)
#--------------------------------------------
# Should this be here ? (Added on 11/03/09)
# Should make code a little faster ?
#--------------------------------------------
C[IDs] = ON_HEAP
if not (SILENT):
print 'Number of pixels on heap = ' + str(n_heap)
print 'Finished with heap insertion.'
print ' '
#----------------------------------
# Prepare to compute neighbor IDs
#----------------------------------
incs = array([-nx - 1, -nx, -nx + 1, -1, 1, nx - 1, nx, nx + 1],
dtype=ID_type)
tstr = str(n_pixels)
#------------------------------------------
# Process pixels to fill pits until there
# are no pixels left on the heap
#------------------------------------------
# NB! Both while loops work for Small,
# KY_Sub, but not Beaver
#-----------------------------------------
while (n_heap > 0):
### while (n_closed < n_pixels):
#--------------------------------------
# Print status message every so often
#--------------------------------------
if ((n_closed % 5000) == 0):
nstr = str(n_closed) + ' of '
if not (SILENT):
print 'n_closed = ' + nstr + tstr
time.sleep(float32(0.001))
#----------------------------------------
# Get pixel in heap with min elevation,
# and remove it from the heap array
#----------------------------------------
if (USE_MY_HEAP):
min_ID = heap.get_min_ID()
zmin = heap.get_min()
else:
pair = heapq.heappop(fast_heap)
min_ID = pair[1]
zmin = pair[0]
## print 'zmin =', zmin
#------------------------------------------
# Flag pixel with min elevation as CLOSED
#------------------------------------------
C[min_ID] = CLOSED
n_closed += 1
#------------------------------------
# Get IDs of this pixel's neighbors
#------------------------------------
ID_vals = (min_ID + incs)
#-------------------------------------------
# Find neighbors with valid IDs that are
# still OPEN (i.e. not(ON_HEAP or CLOSED))
#-------------------------------------------
w = where(logical_and((ID_vals >= 0), (ID_vals < n_pixels)))
if (size(w[0]) != 0):
ID_vals = ID_vals[w[0]]
#-------------------------------------
# Next line excludes neighbor pixels
# that are either ON_HEAP or CLOSED
#-------------------------------------
C_vals = C[ID_vals]
w2 = where(C_vals == OPEN)
if (size(w2[0]) != 0):
ID_vals = ID_vals[w2[0]]
else:
ID_vals = []
#--------------------------------------
# Process each of the neighbor pixels
#--------------------------------------
for ID in ID_vals:
#------------------------------------
# Raise neighbor elevation,
# if it is smaller than zmin & OPEN
#------------------------------------
if (DEM.flat[ID] < zmin):
DEM.flat[ID] = zmin
n_raised += 1
## DEM.flat[ ID ] = numpy.maximum( DEM.flat[ ID ], zmin )
#---------------------------
# Flag neighbor as ON_HEAP
#---------------------------
C[ID] = ON_HEAP
#---------------------------------
# Add neighbor to priority queue
#---------------------------------
if (USE_MY_HEAP):
heap.insert(DEM.flat[ID], ID)
else:
heapq.heappush(fast_heap, (DEM.flat[ID], ID))
if (USE_MY_HEAP):
n_heap = heap.n
else:
n_heap = len(fast_heap)
#---------------------
# Print final report
#---------------------
if not (SILENT):
print 'Total pixels = ' + tstr
print 'Raised pixels = ' + str(n_raised)
print 'Drained pixels = ' + str(n_closed)
if (USE_MY_HEAP):
print 'Max heap size = ' + str(heap.nmax)
print ' '
#-----------------------------------------------------
# Would be better to use CSDMS_base.print_run_time()
#-----------------------------------------------------
run_time = (time.time() - start)
rt_str = ('%10.4f' % run_time) + ' [seconds]'
print 'Run time for fill_pits() = ' + rt_str
print 'Finished with fill_pits().'
print ' '
