def verbosalize_logger(log, verbosity):
if verbosity <= 0:
return
if verbosity >= 8:
while True:
log.critical('MASTML' * random.randint(3, 2**(verbosity - 4)))
def verbosalize_logger(log, verbosity):
if verbosity <= 0:
return
if verbosity >= 8:
while True:
log.critical('MSATML' * random.randint(3, 2**(verbosity - 4)))
old_log = log._log
def new_log(level, msg, *args, **kwargs):
old_log(level, [
None, None, to_upper, to_full_width, to_leet, deep_fry, deep_fry_2,
emojify
][verbosity](msg), *args, **kwargs)
log._log = new_log
def _generic_convert_dem_from_ascii2netcdf(self, name_in, name_out=None, quantity_name=None, verbose=False):
"""Read raster from the following ASCII format (.asc)
Internal function. See public function convert_dem_from_ascii2netcdf
for details.
"""
import os
from anuga.file.netcdf import NetCDFFile
root = name_in[:-4]
# Read Meta data
if verbose:
log.critical("Reading METADATA from %s" % (root + ".prj"))
metadatafile = open(root + ".prj")
metalines = metadatafile.readlines()
metadatafile.close()
L = metalines[0].strip().split()
assert L[0].strip().lower() == "projection"
projection = L[1].strip() # TEXT
L = metalines[1].strip().split()
assert L[0].strip().lower() == "zone"
zone = int(L[1].strip())
L = metalines[2].strip().split()
assert L[0].strip().lower() == "datum"
datum = L[1].strip() # TEXT
L = metalines[3].strip().split()
assert L[0].strip().lower() == "zunits" # IGNORE
zunits = L[1].strip() # TEXT
L = metalines[4].strip().split()
assert L[0].strip().lower() == "units"
units = L[1].strip() # TEXT
L = metalines[5].strip().split()
assert L[0].strip().lower() == "spheroid" # IGNORE
spheroid = L[1].strip() # TEXT
L = metalines[6].strip().split()
assert L[0].strip().lower() == "xshift"
false_easting = float(L[1].strip())
L = metalines[7].strip().split()
assert L[0].strip().lower() == "yshift"
false_northing = float(L[1].strip())
if name_in[-4:] != ".asc":
raise IOError("Input file %s should be of type .asc." % name_in)
# Read DEM data
datafile = open(name_in)
if verbose:
log.critical("Reading raster from %s" % (name_in))
lines = datafile.readlines()
datafile.close()
if verbose:
log.critical("Got %d lines" % len(lines))
ncols = int(lines[0].split()[1].strip())
nrows = int(lines[1].split()[1].strip())
# Do cellsize (line 4) before line 2 and 3
cellsize = float(lines[4].split()[1].strip())
# Checks suggested by Joaquim Luis
# Our internal representation of xllcorner
# and yllcorner is non-standard.
xref = lines[2].split()
if xref[0].strip() == "xllcorner":
xllcorner = float(xref[1].strip()) # + 0.5*cellsize # Correct offset
elif xref[0].strip() == "xllcenter":
xllcorner = float(xref[1].strip())
else:
msg = "Unknown keyword: %s" % xref[0].strip()
raise Exception, msg
yref = lines[3].split()
if yref[0].strip() == "yllcorner":
yllcorner = float(yref[1].strip()) # + 0.5*cellsize # Correct offset
elif yref[0].strip() == "yllcenter":
yllcorner = float(yref[1].strip())
else:
msg = "Unknown keyword: %s" % yref[0].strip()
raise Exception, msg
NODATA_value = int(float(lines[5].split()[1].strip()))
assert len(lines) == nrows + 6
if name_out == None:
netcdfname = name_in[:-4] + ".dem"
else:
netcdfname = name_out + ".dem"
if verbose:
log.critical("Store to NetCDF file %s" % netcdfname)
# NetCDF file definition
fid = NetCDFFile(netcdfname, netcdf_mode_w)
# Create new file
fid.institution = "Geoscience Australia"
fid.description = "NetCDF DEM format for compact and portable storage " "of spatial point data"
fid.ncols = ncols
fid.nrows = nrows
fid.xllcorner = xllcorner
fid.yllcorner = yllcorner
fid.cellsize = cellsize
fid.NODATA_value = NODATA_value
fid.zone = zone
fid.false_easting = false_easting
fid.false_northing = false_northing
fid.projection = projection
fid.datum = datum
fid.units = units
# dimension definitions
fid.createDimension("number_of_rows", nrows)
fid.createDimension("number_of_columns", ncols)
# variable definitions
fid.createVariable(quantity_name, netcdf_float, ("number_of_rows", "number_of_columns"))
# Get handles to the variables
elevation = fid.variables[quantity_name]
# Store data
import numpy
datafile = open(name_in)
elevation[:, :] = numpy.loadtxt(datafile, skiprows=6)
datafile.close()
# n = len(lines[6:])
# for i, line in enumerate(lines[6:]):
# fields = line.split()
# if verbose and i % ((n+10)/10) == 0:
# log.critical('Processing row %d of %d' % (i, nrows))
#
# if len(fields) != ncols:
# msg = 'Wrong number of columns in file "%s" line %d\n' % (name_in, i)
# msg += 'I got %d elements, but there should have been %d\n' % (len(fields), ncols)
# raise Exception, msg
#
# elevation[i, :] = num.array([float(x) for x in fields])
fid.close()
def _generic_dem2pts(
self,
name_in,
name_out=None,
quantity_name=None,
verbose=False,
easting_min=None,
easting_max=None,
northing_min=None,
northing_max=None,
):
"""Read raster from the following NetCDF format (.dem)
Internal function. See public function generic_dem2pts for details.
"""
# FIXME: Can this be written feasibly using write_pts?
import os
from anuga.file.netcdf import NetCDFFile
root = name_in[:-4]
if name_in[-4:] == ".asc":
intermediate = root + ".dem"
if verbose:
log.critical("Preconvert %s from asc to %s" % (name_in, intermediate))
asc2dem(name_in)
name_in = intermediate
elif name_in[-4:] != ".dem":
raise IOError("Input file %s should be of type .asc or .dem." % name_in)
if name_out != None and basename_out[-4:] != ".pts":
raise IOError("Input file %s should be of type .pts." % name_out)
# Get NetCDF
infile = NetCDFFile(name_in, netcdf_mode_r)
if verbose:
log.critical("Reading raster from %s" % (name_in))
ncols = int(infile.ncols)
nrows = int(infile.nrows)
xllcorner = float(infile.xllcorner) # Easting of lower left corner
yllcorner = float(infile.yllcorner) # Northing of lower left corner
cellsize = float(infile.cellsize)
NODATA_value = float(infile.NODATA_value)
dem_elevation = infile.variables[quantity_name]
zone = int(infile.zone)
false_easting = float(infile.false_easting)
false_northing = float(infile.false_northing)
# print ncols, nrows, xllcorner,yllcorner, cellsize, NODATA_value, zone
# Text strings
projection = infile.projection
datum = infile.datum
units = infile.units
# print projection, datum, units
# Get output file
if name_out == None:
ptsname = root + ".pts"
else:
ptsname = name_out
if verbose:
log.critical("Store to NetCDF file %s" % ptsname)
# NetCDF file definition
outfile = NetCDFFile(ptsname, netcdf_mode_w)
# Create new file
outfile.institution = "Geoscience Australia"
outfile.description = "NetCDF pts format for compact and portable " "storage of spatial point data"
# Assign default values
if easting_min is None:
easting_min = xllcorner
if easting_max is None:
easting_max = xllcorner + ncols * cellsize
if northing_min is None:
northing_min = yllcorner
if northing_max is None:
northing_max = yllcorner + nrows * cellsize
# print easting_min, easting_max, northing_min, northing_max
# Compute offsets to update georeferencing
easting_offset = xllcorner - easting_min
northing_offset = yllcorner - northing_min
# Georeferencing
outfile.zone = zone
outfile.xllcorner = easting_min # Easting of lower left corner
outfile.yllcorner = northing_min # Northing of lower left corner
outfile.false_easting = false_easting
outfile.false_northing = false_northing
outfile.projection = projection
outfile.datum = datum
outfile.units = units
# Grid info (FIXME: probably not going to be used, but heck)
outfile.ncols = ncols
outfile.nrows = nrows
dem_elevation_r = num.reshape(dem_elevation, (nrows, ncols))
totalnopoints = nrows * ncols
# ========================================
# Do the preceeding with numpy
# ========================================
y = num.arange(nrows, dtype=num.float)
y = yllcorner + (nrows - 1) * cellsize - y * cellsize
x = num.arange(ncols, dtype=num.float)
x = xllcorner + x * cellsize
xx, yy = num.meshgrid(x, y)
xx = xx.flatten()
yy = yy.flatten()
flag = num.logical_and(
num.logical_and((xx <= easting_max), (xx >= easting_min)),
num.logical_and((yy <= northing_max), (yy >= northing_min)),
)
dem = dem_elevation[:].flatten()
id = num.where(flag)[0]
xx = xx[id]
yy = yy[id]
dem = dem[id]
clippednopoints = len(dem)
# print clippedpoints
# print xx
# print yy
# print dem
data_flag = dem != NODATA_value
data_id = num.where(data_flag)
xx = xx[data_id]
yy = yy[data_id]
dem = dem[data_id]
nn = clippednopoints - len(dem)
nopoints = len(dem)
if verbose:
log.critical("There are %d values in the raster" % totalnopoints)
log.critical("There are %d values in the clipped raster" % clippednopoints)
log.critical("There are %d NODATA_values in the clipped raster" % nn)
outfile.createDimension("number_of_points", nopoints)
outfile.createDimension("number_of_dimensions", 2) # This is 2d data
# Variable definitions
outfile.createVariable("points", netcdf_float, ("number_of_points", "number_of_dimensions"))
outfile.createVariable(quantity_name, netcdf_float, ("number_of_points",))
# Get handles to the variables
points = outfile.variables["points"]
elevation = outfile.variables[quantity_name]
points[:, 0] = xx - easting_min
points[:, 1] = yy - northing_min
elevation[:] = dem
infile.close()
outfile.close()
def _generic_convert_dem_from_ascii2netcdf(self, name_in, name_out = None,
quantity_name = None, verbose = False):
"""
Read raster from the following ASCII format (.asc)
Internal function. See public function convert_dem_from_ascii2netcdf
for details.
"""
import os
from anuga.file.netcdf import NetCDFFile
root = name_in[:-4]
# Read Meta data
if verbose: log.critical('Reading METADATA from %s' % (root + '.prj'))
metadatafile = open(root + '.prj')
metalines = metadatafile.readlines()
metadatafile.close()
L = metalines[0].strip().split()
assert L[0].strip().lower() == 'projection'
projection = L[1].strip() #TEXT
L = metalines[1].strip().split()
assert L[0].strip().lower() == 'zone'
zone = int(L[1].strip())
L = metalines[2].strip().split()
assert L[0].strip().lower() == 'datum'
datum = L[1].strip() #TEXT
L = metalines[3].strip().split()
assert L[0].strip().lower() == 'zunits' #IGNORE
zunits = L[1].strip() #TEXT
L = metalines[4].strip().split()
assert L[0].strip().lower() == 'units'
units = L[1].strip() #TEXT
L = metalines[5].strip().split()
assert L[0].strip().lower() == 'spheroid' #IGNORE
spheroid = L[1].strip() #TEXT
L = metalines[6].strip().split()
assert L[0].strip().lower() == 'xshift'
false_easting = float(L[1].strip())
L = metalines[7].strip().split()
assert L[0].strip().lower() == 'yshift'
false_northing = float(L[1].strip())
if name_in[-4:] != '.asc':
raise IOError('Input file %s should be of type .asc.' % name_in)
#Read DEM data
datafile = open(name_in)
if verbose: log.critical('Reading raster from %s' % (name_in))
lines = datafile.readlines()
datafile.close()
if verbose: log.critical('Got %d lines' % len(lines))
ncols = int(lines[0].split()[1].strip())
nrows = int(lines[1].split()[1].strip())
# Do cellsize (line 4) before line 2 and 3
cellsize = float(lines[4].split()[1].strip())
# Checks suggested by Joaquim Luis
# Our internal representation of xllcorner
# and yllcorner is non-standard.
xref = lines[2].split()
if xref[0].strip() == 'xllcorner':
xllcorner = float(xref[1].strip()) # + 0.5*cellsize # Correct offset
elif xref[0].strip() == 'xllcenter':
xllcorner = float(xref[1].strip())
else:
msg = 'Unknown keyword: %s' % xref[0].strip()
raise Exception, msg
yref = lines[3].split()
if yref[0].strip() == 'yllcorner':
yllcorner = float(yref[1].strip()) # + 0.5*cellsize # Correct offset
elif yref[0].strip() == 'yllcenter':
yllcorner = float(yref[1].strip())
else:
msg = 'Unknown keyword: %s' % yref[0].strip()
raise Exception, msg
NODATA_value = int(float(lines[5].split()[1].strip()))
assert len(lines) == nrows + 6
if name_out == None:
netcdfname = name_in[:-4]+'.dem'
else:
netcdfname = name_out + '.dem'
if verbose: log.critical('Store to NetCDF file %s' % netcdfname)
# NetCDF file definition
fid = NetCDFFile(netcdfname, netcdf_mode_w)
#Create new file
fid.institution = 'Geoscience Australia'
fid.description = 'NetCDF DEM format for compact and portable storage ' \
'of spatial point data'
fid.ncols = ncols
fid.nrows = nrows
fid.xllcorner = xllcorner
fid.yllcorner = yllcorner
fid.cellsize = cellsize
fid.NODATA_value = NODATA_value
fid.zone = zone
fid.false_easting = false_easting
fid.false_northing = false_northing
fid.projection = projection
fid.datum = datum
fid.units = units
# dimension definitions
fid.createDimension('number_of_rows', nrows)
fid.createDimension('number_of_columns', ncols)
# variable definitions
fid.createVariable(quantity_name, netcdf_float, ('number_of_rows',
'number_of_columns'))
# Get handles to the variables
elevation = fid.variables[quantity_name]
#Store data
import numpy
datafile = open(name_in)
elevation[:,:] = numpy.loadtxt(datafile, skiprows=6)
datafile.close()
fid.close()
def _generic_dem2npy(self, name_in, name_out=None, quantity_name=None, verbose=False,
easting_min=None, easting_max=None,
northing_min=None, northing_max=None):
"""
Read raster from the following NetCDF format (.dem)
Internal function. See public function generic_dem2npy for details.
"""
import os
from anuga.file.netcdf import NetCDFFile
root = name_in[:-4]
if name_in[-4:] == '.asc':
intermediate = root + '.dem'
if verbose:
log.critical('Preconvert %s from asc to %s' % \
(name_in, intermediate))
self.generic_asc2dem(name_in)
name_in = intermediate
elif name_in[-4:] != '.dem':
raise IOError('Input file %s should be of type .asc or .dem.' % name_in)
# Get NetCDF
infile = NetCDFFile(name_in, netcdf_mode_r)
if verbose: log.critical('Reading raster from %s' % (name_in))
ncols = int(infile.ncols)
nrows = int(infile.nrows)
xllcorner = float(infile.xllcorner) # Easting of lower left corner
yllcorner = float(infile.yllcorner) # Northing of lower left corner
cellsize = float(infile.cellsize)
NODATA_value = float(infile.NODATA_value)
dem_elevation = infile.variables[quantity_name]
# Assign default values
if easting_min is None: easting_min = xllcorner
if easting_max is None: easting_max = xllcorner + ncols*cellsize
if northing_min is None: northing_min = yllcorner
if northing_max is None: northing_max = yllcorner + nrows*cellsize
#========================================
# Do the preceeding with numpy
#========================================
y = num.arange(nrows,dtype=num.float)
y = yllcorner + (nrows-1)*cellsize - y*cellsize
x = num.arange(ncols,dtype=num.float)
x = xllcorner + x*cellsize
xx,yy = num.meshgrid(x,y)
xx = xx.flatten()
yy = yy.flatten()
flag = num.logical_and(num.logical_and((xx <= easting_max),(xx >= easting_min)),
num.logical_and((yy <= northing_max),(yy >= northing_min)))
dem = dem_elevation[:].flatten()
id = num.where(flag)[0]
xx = xx[id]
yy = yy[id]
dem = dem[id]
data_flag = dem != NODATA_value
data_id = num.where(data_flag)
points = num.zeros((len(data_id[0]),3))
points[:,0] = xx[data_id] - easting_min
points[:,1] = yy[data_id] - northing_min
points[:,2] = dem[data_id]
x_ = num.arange(0, num.ceil(points[:,0].max()))
y_ = num.arange(0, num.ceil(points[:,1].max()))
grid_x, grid_y = num.meshgrid(x_,y_)
grid_z = num.zeros_like(grid_x)
rloc = num.round(points[:,1]).astype(int)
cloc = num.round(points[:,0]).astype(int)
grid_z[rloc, cloc] = points[:,2]
points = num.vstack((grid_x.flatten() + easting_min,
grid_y.flatten() + northing_min,
grid_z.flatten())).T
infile.close()
return points
def _generic_convert_dem_from_ascii2netcdf(self, name_in, name_out = None,
quantity_name = None, verbose = False):
"""
Read raster from the following ASCII format (.asc)
Internal function. See public function convert_dem_from_ascii2netcdf
for details.
"""
import os
from anuga.file.netcdf import NetCDFFile
root = name_in[:-4]
# Read Meta data
if verbose: log.critical('Reading METADATA from %s' % (root + '.prj'))
metadatafile = open(root + '.prj')
metalines = metadatafile.readlines()
metadatafile.close()
L = metalines[0].strip().split()
assert L[0].strip().lower() == 'projection'
projection = L[1].strip() #TEXT
L = metalines[1].strip().split()
assert L[0].strip().lower() == 'zone'
zone = int(L[1].strip())
L = metalines[2].strip().split()
assert L[0].strip().lower() == 'datum'
datum = L[1].strip() #TEXT
L = metalines[3].strip().split()
assert L[0].strip().lower() == 'zunits' #IGNORE
zunits = L[1].strip() #TEXT
L = metalines[4].strip().split()
assert L[0].strip().lower() == 'units'
units = L[1].strip() #TEXT
L = metalines[5].strip().split()
assert L[0].strip().lower() == 'spheroid' #IGNORE
spheroid = L[1].strip() #TEXT
L = metalines[6].strip().split()
assert L[0].strip().lower() == 'xshift'
false_easting = float(L[1].strip())
L = metalines[7].strip().split()
assert L[0].strip().lower() == 'yshift'
false_northing = float(L[1].strip())
if name_in[-4:] != '.asc':
raise IOError('Input file %s should be of type .asc.' % name_in)
#Read DEM data
datafile = open(name_in)
if verbose: log.critical('Reading raster from %s' % (name_in))
lines = datafile.readlines()
datafile.close()
if verbose: log.critical('Got %d lines' % len(lines))
ncols = int(lines[0].split()[1].strip())
nrows = int(lines[1].split()[1].strip())
# Do cellsize (line 4) before line 2 and 3
cellsize = float(lines[4].split()[1].strip())
# Checks suggested by Joaquim Luis
# Our internal representation of xllcorner
# and yllcorner is non-standard.
xref = lines[2].split()
if xref[0].strip() == 'xllcorner':
xllcorner = float(xref[1].strip()) # + 0.5*cellsize # Correct offset
elif xref[0].strip() == 'xllcenter':
xllcorner = float(xref[1].strip())
else:
msg = 'Unknown keyword: %s' % xref[0].strip()
raise Exception, msg
yref = lines[3].split()
if yref[0].strip() == 'yllcorner':
yllcorner = float(yref[1].strip()) # + 0.5*cellsize # Correct offset
elif yref[0].strip() == 'yllcenter':
yllcorner = float(yref[1].strip())
else:
msg = 'Unknown keyword: %s' % yref[0].strip()
raise Exception, msg
NODATA_value = int(float(lines[5].split()[1].strip()))
assert len(lines) == nrows + 6
if name_out == None:
netcdfname = name_in[:-4]+'.dem'
else:
netcdfname = name_out + '.dem'
if verbose: log.critical('Store to NetCDF file %s' % netcdfname)
# NetCDF file definition
fid = NetCDFFile(netcdfname, netcdf_mode_w)
#Create new file
fid.institution = 'Geoscience Australia'
fid.description = 'NetCDF DEM format for compact and portable storage ' \
'of spatial point data'
fid.ncols = ncols
fid.nrows = nrows
fid.xllcorner = xllcorner
fid.yllcorner = yllcorner
fid.cellsize = cellsize
fid.NODATA_value = NODATA_value
fid.zone = zone
fid.false_easting = false_easting
fid.false_northing = false_northing
fid.projection = projection
fid.datum = datum
fid.units = units
# dimension definitions
fid.createDimension('number_of_rows', nrows)
fid.createDimension('number_of_columns', ncols)
# variable definitions
fid.createVariable(quantity_name, netcdf_float, ('number_of_rows',
'number_of_columns'))
# Get handles to the variables
elevation = fid.variables[quantity_name]
#Store data
import numpy
datafile = open(name_in)
elevation[:,:] = numpy.loadtxt(datafile, skiprows=6)
datafile.close()
fid.close()
def _generic_dem2npy(self, name_in, name_out=None, quantity_name=None, verbose=False,
easting_min=None, easting_max=None,
northing_min=None, northing_max=None):
"""
Read raster from the following NetCDF format (.dem)
Internal function. See public function generic_dem2npy for details.
"""
import os
from anuga.file.netcdf import NetCDFFile
root = name_in[:-4]
if name_in[-4:] == '.asc':
intermediate = root + '.dem'
if verbose:
log.critical('Preconvert %s from asc to %s' % \
(name_in, intermediate))
self.generic_asc2dem(name_in)
name_in = intermediate
elif name_in[-4:] != '.dem':
raise IOError('Input file %s should be of type .asc or .dem.' % name_in)
# Get NetCDF
infile = NetCDFFile(name_in, netcdf_mode_r)
if verbose: log.critical('Reading raster from %s' % (name_in))
ncols = int(infile.ncols)
nrows = int(infile.nrows)
xllcorner = float(infile.xllcorner) # Easting of lower left corner
yllcorner = float(infile.yllcorner) # Northing of lower left corner
cellsize = float(infile.cellsize)
NODATA_value = float(infile.NODATA_value)
dem_elevation = infile.variables[quantity_name]
# Assign default values
if easting_min is None: easting_min = xllcorner
if easting_max is None: easting_max = xllcorner + ncols*cellsize
if northing_min is None: northing_min = yllcorner
if northing_max is None: northing_max = yllcorner + nrows*cellsize
#========================================
# Do the preceeding with numpy
#========================================
y = num.arange(nrows,dtype=num.float)
y = yllcorner + (nrows-1)*cellsize - y*cellsize
x = num.arange(ncols,dtype=num.float)
x = xllcorner + x*cellsize
xx,yy = num.meshgrid(x,y)
xx = xx.flatten()
yy = yy.flatten()
flag = num.logical_and(num.logical_and((xx <= easting_max),(xx >= easting_min)),
num.logical_and((yy <= northing_max),(yy >= northing_min)))
dem = dem_elevation[:].flatten()
id = num.where(flag)[0]
xx = xx[id]
yy = yy[id]
dem = dem[id]
data_flag = dem != NODATA_value
data_id = num.where(data_flag)
points = num.zeros((len(data_id[0]),3))
points[:,0] = xx[data_id] - easting_min
points[:,1] = yy[data_id] - northing_min
points[:,2] = dem[data_id]
x_ = num.arange(0, num.ceil(points[:,0].max()))
y_ = num.arange(0, num.ceil(points[:,1].max()))
grid_x, grid_y = num.meshgrid(x_,y_)
grid_z = num.zeros_like(grid_x)
rloc = num.round(points[:,1]).astype(int)
cloc = num.round(points[:,0]).astype(int)
grid_z[rloc, cloc] = points[:,2]
points = num.vstack((grid_x.flatten() + easting_min,
grid_y.flatten() + northing_min,
grid_z.flatten())).T
infile.close()
return points
