def ferret2sww(basename_in, name_out=None,
verbose=False,
minlat=None, maxlat=None,
minlon=None, maxlon=None,
mint=None, maxt=None, mean_stage=0,
origin=None, zscale=1,
fail_on_NaN=True,
NaN_filler=0,
elevation=None,
inverted_bathymetry=True
): #FIXME: Bathymetry should be obtained
#from MOST somehow.
#Alternatively from elsewhere
#or, as a last resort,
#specified here.
#The value of -100 will work
#for the Wollongong tsunami
#scenario but is very hacky
"""Convert MOST and 'Ferret' NetCDF format for wave propagation to
sww format native to abstract_2d_finite_volumes.
Specify only basename_in and read files of the form
basefilename_ha.nc, basefilename_ua.nc, basefilename_va.nc containing
relative height, x-velocity and y-velocity, respectively.
Also convert latitude and longitude to UTM. All coordinates are
assumed to be given in the GDA94 datum.
min's and max's: If omitted - full extend is used.
To include a value min may equal it, while max must exceed it.
Lat and lon are assuemd to be in decimal degrees
origin is a 3-tuple with geo referenced
UTM coordinates (zone, easting, northing)
nc format has values organised as HA[TIME, LATITUDE, LONGITUDE]
which means that longitude is the fastest
varying dimension (row major order, so to speak)
ferret2sww uses grid points as vertices in a triangular grid
counting vertices from lower left corner upwards, then right
"""
from anuga.file.netcdf import NetCDFFile
_assert_lat_long(minlat, maxlat, minlon, maxlon)
if name_out != None and name_out[-4:] != '.sww':
raise IOError('Output file %s should be of type .sww.' % name_out)
# Get NetCDF data
if verbose: log.critical('Reading files %s_*.nc' % basename_in)
# Wave amplitude (cm)
file_h = NetCDFFile(basename_in + '_ha.nc', netcdf_mode_r)
# Velocity (x) (cm/s)
file_u = NetCDFFile(basename_in + '_ua.nc', netcdf_mode_r)
# Velocity (y) (cm/s)
file_v = NetCDFFile(basename_in + '_va.nc', netcdf_mode_r)
# Elevation (z) (m)
file_e = NetCDFFile(basename_in + '_e.nc', netcdf_mode_r)
if name_out is None:
swwname = basename_in + '.sww'
else:
swwname = name_out
# Get dimensions of file_h
for dimension in file_h.dimensions.keys():
if dimension[:3] == 'LON':
dim_h_longitude = dimension
if dimension[:3] == 'LAT':
dim_h_latitude = dimension
if dimension[:4] == 'TIME':
dim_h_time = dimension
times = file_h.variables[dim_h_time]
latitudes = file_h.variables[dim_h_latitude]
longitudes = file_h.variables[dim_h_longitude]
kmin, kmax, lmin, lmax = get_min_max_indices(latitudes[:],
longitudes[:],
minlat, maxlat,
minlon, maxlon)
# get dimensions for file_e
for dimension in file_e.dimensions.keys():
if dimension[:3] == 'LON':
dim_e_longitude = dimension
if dimension[:3] == 'LAT':
dim_e_latitude = dimension
# get dimensions for file_u
for dimension in file_u.dimensions.keys():
if dimension[:3] == 'LON':
dim_u_longitude = dimension
if dimension[:3] == 'LAT':
dim_u_latitude = dimension
# get dimensions for file_v
for dimension in file_v.dimensions.keys():
if dimension[:3] == 'LON':
dim_v_longitude = dimension
if dimension[:3] == 'LAT':
dim_v_latitude = dimension
# Precision used by most for lat/lon is 4 or 5 decimals
e_lat = num.around(file_e.variables[dim_e_latitude][:], 5)
e_lon = num.around(file_e.variables[dim_e_longitude][:], 5)
# Check that files are compatible
assert num.allclose(latitudes, file_u.variables[dim_u_latitude])
assert num.allclose(latitudes, file_v.variables[dim_v_latitude])
assert num.allclose(latitudes, e_lat)
assert num.allclose(longitudes, file_u.variables[dim_u_longitude])
assert num.allclose(longitudes, file_v.variables[dim_v_longitude])
assert num.allclose(longitudes, e_lon)
if mint is None:
jmin = 0
mint = times[0]
else:
jmin = num.searchsorted(times, mint)
# numpy.int32 didn't work in slicing of amplitude below
jmin = int(jmin)
if maxt is None:
jmax = len(times)
maxt = times[-1]
else:
jmax = num.searchsorted(times, maxt)
# numpy.int32 didn't work in slicing of amplitude below
jmax = int(jmax)
kmin, kmax, lmin, lmax = get_min_max_indices(latitudes[:],
longitudes[:],
minlat, maxlat,
minlon, maxlon)
times = times[jmin:jmax]
latitudes = latitudes[kmin:kmax]
longitudes = longitudes[lmin:lmax]
if verbose: log.critical('cropping')
zname = 'ELEVATION'
amplitudes = file_h.variables['HA'][jmin:jmax, kmin:kmax, lmin:lmax]
uspeed = file_u.variables['UA'][jmin:jmax, kmin:kmax, lmin:lmax] #Lon
vspeed = file_v.variables['VA'][jmin:jmax, kmin:kmax, lmin:lmax] #Lat
elevations = file_e.variables[zname][kmin:kmax, lmin:lmax]
# Get missing values
nan_ha = file_h.variables['HA'].missing_value
nan_ua = file_u.variables['UA'].missing_value
nan_va = file_v.variables['VA'].missing_value
if hasattr(file_e.variables[zname],'missing_value'):
nan_e = file_e.variables[zname].missing_value
else:
nan_e = None
# Cleanup
missing = (amplitudes == nan_ha)
if num.sometrue (missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_ha.nc'
raise DataMissingValuesError, msg
else:
amplitudes = amplitudes*(missing==0) + missing*NaN_filler
missing = (uspeed == nan_ua)
if num.sometrue (missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_ua.nc'
raise DataMissingValuesError, msg
else:
uspeed = uspeed*(missing==0) + missing*NaN_filler
missing = (vspeed == nan_va)
if num.sometrue (missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_va.nc'
raise DataMissingValuesError, msg
else:
vspeed = vspeed*(missing==0) + missing*NaN_filler
missing = (elevations == nan_e)
if num.sometrue (missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_e.nc'
raise DataMissingValuesError, msg
else:
elevations = elevations*(missing==0) + missing*NaN_filler
number_of_times = times.shape[0]
number_of_latitudes = latitudes.shape[0]
number_of_longitudes = longitudes.shape[0]
assert amplitudes.shape[0] == number_of_times
assert amplitudes.shape[1] == number_of_latitudes
assert amplitudes.shape[2] == number_of_longitudes
if verbose:
_show_stats((latitudes, longitudes), times, amplitudes, \
(uspeed, vspeed), elevations)
# print number_of_latitudes, number_of_longitudes
number_of_points = number_of_latitudes * number_of_longitudes
number_of_volumes = (number_of_latitudes-1) * (number_of_longitudes-1) * 2
file_h.close()
file_u.close()
file_v.close()
file_e.close()
# NetCDF file definition
outfile = NetCDFFile(swwname, netcdf_mode_w)
description = 'Converted from Ferret files: %s, %s, %s, %s' \
% (basename_in + '_ha.nc',
basename_in + '_ua.nc',
basename_in + '_va.nc',
basename_in + '_e.nc')
# Create new file
starttime = times[0]
sww = Write_sww(['elevation'], ['stage', 'xmomentum', 'ymomentum'])
sww.store_header(outfile, times, number_of_volumes,
number_of_points, description=description,
verbose=verbose, sww_precision=netcdf_float)
# Store
from anuga.coordinate_transforms.redfearn import redfearn
x = num.zeros(number_of_points, num.float) #Easting
y = num.zeros(number_of_points, num.float) #Northing
if verbose:
log.critical('Making triangular grid')
# Check zone boundaries
refzone, _, _ = redfearn(latitudes[0], longitudes[0])
vertices = {}
i = 0
for k, lat in enumerate(latitudes): # Y direction
for l, lon in enumerate(longitudes): # X direction
vertices[l, k] = i
_, easting, northing = redfearn(lat, lon)
#msg = 'Zone boundary crossed at longitude =', lon
#assert zone == refzone, msg
#print '%7.2f %7.2f %8.2f %8.2f' %(lon, lat, easting, northing)
x[i] = easting
y[i] = northing
i += 1
#Construct 2 triangles per 'rectangular' element
volumes = []
for l in range(number_of_longitudes-1): # X direction
for k in range(number_of_latitudes-1): # Y direction
v1 = vertices[l, k+1]
v2 = vertices[l, k]
v3 = vertices[l+1, k+1]
v4 = vertices[l+1, k]
volumes.append([v1, v2, v3]) #Upper element
volumes.append([v4, v3, v2]) #Lower element
volumes = num.array(volumes, num.int) #array default#
if origin is None:
origin = Geo_reference(refzone, min(x), min(y))
geo_ref = write_NetCDF_georeference(origin, outfile)
if elevation is not None:
z = elevation
else:
if inverted_bathymetry:
z = -1 * elevations
else:
z = elevations
#FIXME: z should be obtained from MOST and passed in here
#FIXME use the Write_sww instance(sww) to write this info
z = num.resize(z, outfile.variables['elevation'][:].shape)
outfile.variables['x'][:] = x - geo_ref.get_xllcorner()
outfile.variables['y'][:] = y - geo_ref.get_yllcorner()
#outfile.variables['z'][:] = z #FIXME HACK for bacwards compat.
outfile.variables['elevation'][:] = z
outfile.variables['volumes'][:] = volumes.astype(num.int32) #For Opteron 64
#Time stepping
stage = outfile.variables['stage']
xmomentum = outfile.variables['xmomentum']
ymomentum = outfile.variables['ymomentum']
if verbose:
log.critical('Converting quantities')
n = len(times)
for j in range(n):
if verbose and j % ((n+10)/10) == 0:
log.critical(' Doing %d of %d' % (j, n))
i = 0
for k in range(number_of_latitudes): # Y direction
for l in range(number_of_longitudes): # X direction
w = zscale * amplitudes[j, k, l] / 100 + mean_stage
stage[j, i] = w
h = w - z[i]
xmomentum[j, i] = uspeed[j, k, l]/100*h
ymomentum[j, i] = vspeed[j, k, l]/100*h
i += 1
#outfile.close()
#FIXME: Refactor using code from file_function.statistics
#Something like print swwstats(swwname)
if verbose:
time_info = times, starttime, mint, maxt
_show_sww_stats(outfile, swwname, geo_ref, time_info)
outfile.close()
def urs2sts(basename_in,
basename_out=None,
weights=None,
verbose=False,
origin=None,
zone=None,
central_meridian=None,
mean_stage=0.0,
zscale=1.0,
ordering_filename=None):
"""Convert URS mux2 format for wave propagation to sts format
Also convert latitude and longitude to UTM. All coordinates are
assumed to be given in the GDA94 datum
origin is a 3-tuple with geo referenced
UTM coordinates (zone, easting, northing)
inputs:
basename_in: list of source file prefixes
These are combined with the extensions:
WAVEHEIGHT_MUX2_LABEL = '-z-mux2' for stage
EAST_VELOCITY_MUX2_LABEL = '-e-mux2' xmomentum
NORTH_VELOCITY_MUX2_LABEL = '-n-mux2' and ymomentum
to create a 2D list of mux2 file. The rows are associated with each
quantity and must have the above extensions
the columns are the list of file prefixes.
ordering: a .txt file name specifying which mux2 gauge points are
to be stored. This is indicated by the index of the gauge
in the ordering file.
ordering file format:
1st line: 'index,longitude,latitude\n'
other lines: index,longitude,latitude
If ordering is None or ordering file is empty then
all points are taken in the order they
appear in the mux2 file.
output:
basename_out: name of sts file in which mux2 data is stored.
NOTE: South is positive in mux files so sign of y-component of velocity is reverted
"""
import os
from anuga.file.netcdf import NetCDFFile
from operator import __and__
if not isinstance(basename_in, list):
if verbose: log.critical('Reading single source')
basename_in = [basename_in]
# This is the value used in the mux file format to indicate NAN data
# FIXME (Ole): This should be changed everywhere to IEEE NAN when
# we upgrade to Numpy
NODATA = 99
# Check that basename is a list of strings
if not reduce(__and__, map(lambda z: isinstance(z, basestring),
basename_in)):
msg = 'basename_in must be a string or list of strings'
raise Exception, msg
# Find the number of sources to be used
numSrc = len(basename_in)
# A weight must be specified for each source
if weights is None:
# Default is equal weighting
weights = num.ones(numSrc, num.float) / numSrc
else:
weights = ensure_numeric(weights)
msg = 'When combining multiple sources must specify a weight for ' \
'mux2 source file'
assert len(weights) == numSrc, msg
if verbose: log.critical('Weights used in urs2sts: %s' % str(weights))
# Check output filename
if basename_out is None:
msg = 'STS filename must be specified as basename_out ' \
'in function urs2sts'
raise Exception, msg
if basename_out.endswith('.sts'):
stsname = basename_out
else:
stsname = basename_out + '.sts'
# Create input filenames from basenames and check their existence
files_in = [[], [], []]
for files in basename_in:
files_in[0].append(files + WAVEHEIGHT_MUX2_LABEL),
files_in[1].append(files + EAST_VELOCITY_MUX2_LABEL)
files_in[2].append(files + NORTH_VELOCITY_MUX2_LABEL)
quantities = ['HA', 'UA', 'VA'] # Quantity names used in the MUX2 format
for i in range(len(quantities)):
for file_in in files_in[i]:
if (os.access(file_in, os.R_OK) == 0):
msg = 'File %s does not exist or is not accessible' % file_in
raise IOError, msg
# Establish permutation array
if ordering_filename is not None:
if verbose is True:
log.critical('Reading ordering file %s' % ordering_filename)
# Read ordering file
try:
fid = open(ordering_filename, 'r')
file_header = fid.readline().split(',')
ordering_lines = fid.readlines()
fid.close()
except:
msg = 'Cannot open %s' % ordering_filename
raise Exception, msg
reference_header = 'index, longitude, latitude\n'
reference_header_split = reference_header.split(',')
for i in range(3):
if not file_header[i].strip() == reference_header_split[i].strip():
msg = 'File must contain header: ' + reference_header
raise Exception, msg
if len(ordering_lines) < 2:
msg = 'File must contain at least two points'
raise Exception, msg
permutation = [int(line.split(',')[0]) for line in ordering_lines]
permutation = ensure_numeric(permutation)
else:
permutation = None
# Read MUX2 files
if (verbose): log.critical('reading mux2 file')
mux = {}
times_old = 0.0
latitudes_old = 0.0
longitudes_old = 0.0
elevation_old = 0.0
starttime_old = 0.0
for i, quantity in enumerate(quantities):
# For each quantity read the associated list of source mux2 file with
# extention associated with that quantity
times, latitudes, longitudes, elevation, mux[quantity], starttime \
= read_mux2_py(files_in[i], weights, permutation, verbose=verbose)
# Check that all quantities have consistent time and space information
if quantity != quantities[0]:
msg = '%s, %s and %s have inconsistent gauge data' \
% (files_in[0], files_in[1], files_in[2])
assert num.allclose(times, times_old), msg
assert num.allclose(latitudes, latitudes_old), msg
assert num.allclose(longitudes, longitudes_old), msg
assert num.allclose(elevation, elevation_old), msg
assert num.allclose(starttime, starttime_old), msg
times_old = times
latitudes_old = latitudes
longitudes_old = longitudes
elevation_old = elevation
starttime_old = starttime
# Self check - can be removed to improve speed
#ref_longitudes = [float(line.split(',')[1]) for line in ordering_lines]
#ref_latitudes = [float(line.split(',')[2]) for line in ordering_lines]
#
#msg = 'Longitudes specified in ordering file do not match those ' \
# 'found in mux files. ' \
# 'I got %s instead of %s (only beginning shown)' \
# % (str(longitudes[:10]) + '...',
# str(ref_longitudes[:10]) + '...')
#assert allclose(longitudes, ref_longitudes), msg
#
#msg = 'Latitudes specified in ordering file do not match those ' \
# 'found in mux files. '
# 'I got %s instead of %s (only beginning shown)' \
# % (str(latitudes[:10]) + '...',
# str(ref_latitudes[:10]) + '...')
#assert allclose(latitudes, ref_latitudes), msg
# Store timeseries in STS file
msg = 'File is empty and or clipped region not in file region'
assert len(latitudes > 0), msg
number_of_points = latitudes.shape[0] # Number of stations retrieved
number_of_times = times.shape[0] # Number of timesteps
number_of_latitudes = latitudes.shape[0] # Number latitudes
number_of_longitudes = longitudes.shape[0] # Number longitudes
# The permutation vector of contains original indices
# as given in ordering file or None in which case points
# are assigned the trivial indices enumerating them from
# 0 to number_of_points-1
if permutation is None:
permutation = num.arange(number_of_points, dtype=num.int)
# NetCDF file definition
outfile = NetCDFFile(stsname, netcdf_mode_w)
description = 'Converted from URS mux2 files: %s' % basename_in
# Create new file
sts = Write_sts()
sts.store_header(outfile,
times + starttime,
number_of_points,
description=description,
verbose=verbose,
sts_precision=netcdf_float)
# Store
from anuga.coordinate_transforms.redfearn import redfearn
x = num.zeros(number_of_points, num.float) # Easting
y = num.zeros(number_of_points, num.float) # Northing
# Check zone boundaries
if zone is None:
refzone, _, _ = redfearn(latitudes[0],
longitudes[0],
central_meridian=central_meridian)
else:
refzone = zone
old_zone = refzone
old_easting = 0.0
old_northing = 0.0
for i in range(number_of_points):
computed_zone, easting, northing = redfearn(
latitudes[i],
longitudes[i],
zone=zone,
central_meridian=central_meridian)
x[i] = easting
y[i] = northing
if computed_zone != refzone:
msg = 'All sts gauges need to be in the same zone. \n'
msg += 'offending gauge:Zone %d,%.4f, %4f\n' \
% (computed_zone, easting, northing)
msg += 'previous gauge:Zone %d,%.4f, %4f' \
% (old_zone, old_easting, old_northing)
raise Exception, msg
old_zone = computed_zone
old_easting = easting
old_northing = northing
if origin is None:
origin = Geo_reference(refzone, min(x), min(y))
geo_ref = write_NetCDF_georeference(origin, outfile)
elevation = num.resize(elevation, outfile.variables['elevation'][:].shape)
outfile.variables['permutation'][:] = permutation.astype(
num.int32) # Opteron 64
outfile.variables['x'][:] = x - geo_ref.get_xllcorner()
outfile.variables['y'][:] = y - geo_ref.get_yllcorner()
outfile.variables['elevation'][:] = elevation
stage = outfile.variables['stage']
xmomentum = outfile.variables['xmomentum']
ymomentum = outfile.variables['ymomentum']
if verbose: log.critical('Converting quantities')
for j in range(len(times)):
for i in range(number_of_points):
ha = mux['HA'][i, j]
ua = mux['UA'][i, j]
va = mux['VA'][i, j]
if ha == NODATA:
if verbose:
msg = 'Setting nodata value %d to 0 at time = %f, ' \
'point = %d' % (ha, times[j], i)
log.critical(msg)
ha = 0.0
ua = 0.0
va = 0.0
w = zscale * ha + mean_stage
h = w - elevation[i]
stage[j, i] = w
xmomentum[j, i] = ua * h
ymomentum[j, i] = -va * h # South is positive in mux files
outfile.close()
if verbose:
log.critical('Wrote sts file ' + stsname)
def urs2sts(basename_in, basename_out=None,
weights=None,
verbose=False,
origin=None,
zone=None,
central_meridian=None,
mean_stage=0.0,
zscale=1.0,
ordering_filename=None):
"""Convert URS mux2 format for wave propagation to sts format
Also convert latitude and longitude to UTM. All coordinates are
assumed to be given in the GDA94 datum
origin is a 3-tuple with geo referenced
UTM coordinates (zone, easting, northing)
inputs:
basename_in: list of source file prefixes
These are combined with the extensions:
WAVEHEIGHT_MUX2_LABEL = '-z-mux2' for stage
EAST_VELOCITY_MUX2_LABEL = '-e-mux2' xmomentum
NORTH_VELOCITY_MUX2_LABEL = '-n-mux2' and ymomentum
to create a 2D list of mux2 file. The rows are associated with each
quantity and must have the above extensions
the columns are the list of file prefixes.
ordering: a .txt file name specifying which mux2 gauge points are
to be stored. This is indicated by the index of the gauge
in the ordering file.
ordering file format:
1st line: 'index,longitude,latitude\n'
other lines: index,longitude,latitude
If ordering is None or ordering file is empty then
all points are taken in the order they
appear in the mux2 file.
output:
basename_out: name of sts file in which mux2 data is stored.
NOTE: South is positive in mux files so sign of y-component of velocity is reverted
"""
import os
from anuga.file.netcdf import NetCDFFile
from operator import __and__
if not isinstance(basename_in, list):
if verbose: log.critical('Reading single source')
basename_in = [basename_in]
# This is the value used in the mux file format to indicate NAN data
# FIXME (Ole): This should be changed everywhere to IEEE NAN when
# we upgrade to Numpy
NODATA = 99
# Check that basename is a list of strings
if not reduce(__and__, map(lambda z:isinstance(z,basestring), basename_in)):
msg= 'basename_in must be a string or list of strings'
raise Exception, msg
# Find the number of sources to be used
numSrc = len(basename_in)
# A weight must be specified for each source
if weights is None:
# Default is equal weighting
weights = num.ones(numSrc, num.float) / numSrc
else:
weights = ensure_numeric(weights)
msg = 'When combining multiple sources must specify a weight for ' \
'mux2 source file'
assert len(weights) == numSrc, msg
if verbose: log.critical('Weights used in urs2sts: %s' % str(weights))
# Check output filename
if basename_out is None:
msg = 'STS filename must be specified as basename_out ' \
'in function urs2sts'
raise Exception, msg
if basename_out.endswith('.sts'):
stsname = basename_out
else:
stsname = basename_out + '.sts'
# Create input filenames from basenames and check their existence
files_in = [[], [], []]
for files in basename_in:
files_in[0].append(files + WAVEHEIGHT_MUX2_LABEL),
files_in[1].append(files + EAST_VELOCITY_MUX2_LABEL)
files_in[2].append(files + NORTH_VELOCITY_MUX2_LABEL)
quantities = ['HA','UA','VA'] # Quantity names used in the MUX2 format
for i in range(len(quantities)):
for file_in in files_in[i]:
if (os.access(file_in, os.R_OK) == 0):
msg = 'File %s does not exist or is not accessible' % file_in
raise IOError, msg
# Establish permutation array
if ordering_filename is not None:
if verbose is True: log.critical('Reading ordering file %s'
% ordering_filename)
# Read ordering file
try:
fid = open(ordering_filename, 'r')
file_header = fid.readline().split(',')
ordering_lines = fid.readlines()
fid.close()
except:
msg = 'Cannot open %s' % ordering_filename
raise Exception, msg
reference_header = 'index, longitude, latitude\n'
reference_header_split = reference_header.split(',')
for i in range(3):
if not file_header[i].strip() == reference_header_split[i].strip():
msg = 'File must contain header: ' + reference_header
raise Exception, msg
if len(ordering_lines) < 2:
msg = 'File must contain at least two points'
raise Exception, msg
permutation = [int(line.split(',')[0]) for line in ordering_lines]
permutation = ensure_numeric(permutation)
else:
permutation = None
# Read MUX2 files
if (verbose): log.critical('reading mux2 file')
mux={}
times_old = 0.0
latitudes_old = 0.0
longitudes_old = 0.0
elevation_old = 0.0
starttime_old = 0.0
for i, quantity in enumerate(quantities):
# For each quantity read the associated list of source mux2 file with
# extention associated with that quantity
times, latitudes, longitudes, elevation, mux[quantity], starttime \
= read_mux2_py(files_in[i], weights, permutation, verbose=verbose)
# Check that all quantities have consistent time and space information
if quantity != quantities[0]:
msg = '%s, %s and %s have inconsistent gauge data' \
% (files_in[0], files_in[1], files_in[2])
assert num.allclose(times, times_old), msg
assert num.allclose(latitudes, latitudes_old), msg
assert num.allclose(longitudes, longitudes_old), msg
assert num.allclose(elevation, elevation_old), msg
assert num.allclose(starttime, starttime_old), msg
times_old = times
latitudes_old = latitudes
longitudes_old = longitudes
elevation_old = elevation
starttime_old = starttime
# Self check - can be removed to improve speed
#ref_longitudes = [float(line.split(',')[1]) for line in ordering_lines]
#ref_latitudes = [float(line.split(',')[2]) for line in ordering_lines]
#
#msg = 'Longitudes specified in ordering file do not match those ' \
# 'found in mux files. ' \
# 'I got %s instead of %s (only beginning shown)' \
# % (str(longitudes[:10]) + '...',
# str(ref_longitudes[:10]) + '...')
#assert allclose(longitudes, ref_longitudes), msg
#
#msg = 'Latitudes specified in ordering file do not match those ' \
# 'found in mux files. '
# 'I got %s instead of %s (only beginning shown)' \
# % (str(latitudes[:10]) + '...',
# str(ref_latitudes[:10]) + '...')
#assert allclose(latitudes, ref_latitudes), msg
# Store timeseries in STS file
msg = 'File is empty and or clipped region not in file region'
assert len(latitudes > 0), msg
number_of_points = latitudes.shape[0] # Number of stations retrieved
number_of_times = times.shape[0] # Number of timesteps
number_of_latitudes = latitudes.shape[0] # Number latitudes
number_of_longitudes = longitudes.shape[0] # Number longitudes
# The permutation vector of contains original indices
# as given in ordering file or None in which case points
# are assigned the trivial indices enumerating them from
# 0 to number_of_points-1
if permutation is None:
permutation = num.arange(number_of_points, dtype=num.int)
# NetCDF file definition
outfile = NetCDFFile(stsname, netcdf_mode_w)
description = 'Converted from URS mux2 files: %s' % basename_in
# Create new file
sts = Write_sts()
sts.store_header(outfile,
times+starttime,
number_of_points,
description=description,
verbose=verbose,
sts_precision=netcdf_float)
# Store
from anuga.coordinate_transforms.redfearn import redfearn
x = num.zeros(number_of_points, num.float) # Easting
y = num.zeros(number_of_points, num.float) # Northing
# Check zone boundaries
if zone is None:
refzone, _, _ = redfearn(latitudes[0], longitudes[0],
central_meridian=central_meridian)
else:
refzone = zone
old_zone = refzone
old_easting = 0.0
old_northing = 0.0
for i in range(number_of_points):
computed_zone, easting, northing = redfearn(latitudes[i], longitudes[i],
zone=zone,
central_meridian=central_meridian)
x[i] = easting
y[i] = northing
if computed_zone != refzone:
msg = 'All sts gauges need to be in the same zone. \n'
msg += 'offending gauge:Zone %d,%.4f, %4f\n' \
% (computed_zone, easting, northing)
msg += 'previous gauge:Zone %d,%.4f, %4f' \
% (old_zone, old_easting, old_northing)
raise Exception, msg
old_zone = computed_zone
old_easting = easting
old_northing = northing
if origin is None:
origin = Geo_reference(refzone, min(x), min(y))
geo_ref = write_NetCDF_georeference(origin, outfile)
elevation = num.resize(elevation, outfile.variables['elevation'][:].shape)
outfile.variables['permutation'][:] = permutation.astype(num.int32) # Opteron 64
outfile.variables['x'][:] = x - geo_ref.get_xllcorner()
outfile.variables['y'][:] = y - geo_ref.get_yllcorner()
outfile.variables['elevation'][:] = elevation
stage = outfile.variables['stage']
xmomentum = outfile.variables['xmomentum']
ymomentum = outfile.variables['ymomentum']
if verbose: log.critical('Converting quantities')
for j in range(len(times)):
for i in range(number_of_points):
ha = mux['HA'][i,j]
ua = mux['UA'][i,j]
va = mux['VA'][i,j]
if ha == NODATA:
if verbose:
msg = 'Setting nodata value %d to 0 at time = %f, ' \
'point = %d' % (ha, times[j], i)
log.critical(msg)
ha = 0.0
ua = 0.0
va = 0.0
w = zscale*ha + mean_stage
h = w - elevation[i]
stage[j,i] = w
xmomentum[j,i] = ua * h
ymomentum[j,i] = -va * h # South is positive in mux files
outfile.close()
if verbose:
log.critical('Wrote sts file ' + stsname)
def ferret2sww(
basename_in,
name_out=None,
verbose=False,
minlat=None,
maxlat=None,
minlon=None,
maxlon=None,
mint=None,
maxt=None,
mean_stage=0,
origin=None,
zscale=1,
fail_on_NaN=True,
NaN_filler=0,
elevation=None,
inverted_bathymetry=True): #FIXME: Bathymetry should be obtained
#from MOST somehow.
#Alternatively from elsewhere
#or, as a last resort,
#specified here.
#The value of -100 will work
#for the Wollongong tsunami
#scenario but is very hacky
"""Convert MOST and 'Ferret' NetCDF format for wave propagation to
sww format native to abstract_2d_finite_volumes.
Specify only basename_in and read files of the form
basefilename_ha.nc, basefilename_ua.nc, basefilename_va.nc containing
relative height, x-velocity and y-velocity, respectively.
Also convert latitude and longitude to UTM. All coordinates are
assumed to be given in the GDA94 datum.
min's and max's: If omitted - full extend is used.
To include a value min may equal it, while max must exceed it.
Lat and lon are assuemd to be in decimal degrees
origin is a 3-tuple with geo referenced
UTM coordinates (zone, easting, northing)
nc format has values organised as HA[TIME, LATITUDE, LONGITUDE]
which means that longitude is the fastest
varying dimension (row major order, so to speak)
ferret2sww uses grid points as vertices in a triangular grid
counting vertices from lower left corner upwards, then right
"""
from anuga.file.netcdf import NetCDFFile
_assert_lat_long(minlat, maxlat, minlon, maxlon)
if name_out != None and name_out[-4:] != '.sww':
raise IOError('Output file %s should be of type .sww.' % name_out)
# Get NetCDF data
if verbose: log.critical('Reading files %s_*.nc' % basename_in)
# Wave amplitude (cm)
file_h = NetCDFFile(basename_in + '_ha.nc', netcdf_mode_r)
# Velocity (x) (cm/s)
file_u = NetCDFFile(basename_in + '_ua.nc', netcdf_mode_r)
# Velocity (y) (cm/s)
file_v = NetCDFFile(basename_in + '_va.nc', netcdf_mode_r)
# Elevation (z) (m)
file_e = NetCDFFile(basename_in + '_e.nc', netcdf_mode_r)
if name_out is None:
swwname = basename_in + '.sww'
else:
swwname = name_out
# Get dimensions of file_h
for dimension in list(file_h.dimensions.keys()):
if dimension[:3] == 'LON':
dim_h_longitude = dimension
if dimension[:3] == 'LAT':
dim_h_latitude = dimension
if dimension[:4] == 'TIME':
dim_h_time = dimension
times = file_h.variables[dim_h_time]
latitudes = file_h.variables[dim_h_latitude]
longitudes = file_h.variables[dim_h_longitude]
kmin, kmax, lmin, lmax = get_min_max_indices(latitudes[:], longitudes[:],
minlat, maxlat, minlon,
maxlon)
# get dimensions for file_e
for dimension in list(file_e.dimensions.keys()):
if dimension[:3] == 'LON':
dim_e_longitude = dimension
if dimension[:3] == 'LAT':
dim_e_latitude = dimension
# get dimensions for file_u
for dimension in list(file_u.dimensions.keys()):
if dimension[:3] == 'LON':
dim_u_longitude = dimension
if dimension[:3] == 'LAT':
dim_u_latitude = dimension
# get dimensions for file_v
for dimension in list(file_v.dimensions.keys()):
if dimension[:3] == 'LON':
dim_v_longitude = dimension
if dimension[:3] == 'LAT':
dim_v_latitude = dimension
# Precision used by most for lat/lon is 4 or 5 decimals
e_lat = num.around(file_e.variables[dim_e_latitude][:], 5)
e_lon = num.around(file_e.variables[dim_e_longitude][:], 5)
# Check that files are compatible
assert num.allclose(latitudes, file_u.variables[dim_u_latitude])
assert num.allclose(latitudes, file_v.variables[dim_v_latitude])
assert num.allclose(latitudes, e_lat)
assert num.allclose(longitudes, file_u.variables[dim_u_longitude])
assert num.allclose(longitudes, file_v.variables[dim_v_longitude])
assert num.allclose(longitudes, e_lon)
if mint is None:
jmin = 0
mint = times[0]
else:
jmin = num.searchsorted(times, mint)
# numpy.int32 didn't work in slicing of amplitude below
jmin = int(jmin)
if maxt is None:
jmax = len(times)
maxt = times[-1]
else:
jmax = num.searchsorted(times, maxt)
# numpy.int32 didn't work in slicing of amplitude below
jmax = int(jmax)
kmin, kmax, lmin, lmax = get_min_max_indices(latitudes[:], longitudes[:],
minlat, maxlat, minlon,
maxlon)
times = times[jmin:jmax]
latitudes = latitudes[kmin:kmax]
longitudes = longitudes[lmin:lmax]
if verbose: log.critical('cropping')
zname = 'ELEVATION'
amplitudes = file_h.variables['HA'][jmin:jmax, kmin:kmax, lmin:lmax]
uspeed = file_u.variables['UA'][jmin:jmax, kmin:kmax, lmin:lmax] #Lon
vspeed = file_v.variables['VA'][jmin:jmax, kmin:kmax, lmin:lmax] #Lat
elevations = file_e.variables[zname][kmin:kmax, lmin:lmax]
# Get missing values
nan_ha = file_h.variables['HA'].missing_value
nan_ua = file_u.variables['UA'].missing_value
nan_va = file_v.variables['VA'].missing_value
if hasattr(file_e.variables[zname], 'missing_value'):
nan_e = file_e.variables[zname].missing_value
else:
nan_e = None
# Cleanup
missing = (amplitudes == nan_ha)
if num.sometrue(missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_ha.nc'
raise_(DataMissingValuesError, msg)
else:
amplitudes = amplitudes * (missing == 0) + missing * NaN_filler
missing = (uspeed == nan_ua)
if num.sometrue(missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_ua.nc'
raise_(DataMissingValuesError, msg)
else:
uspeed = uspeed * (missing == 0) + missing * NaN_filler
missing = (vspeed == nan_va)
if num.sometrue(missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_va.nc'
raise_(DataMissingValuesError, msg)
else:
vspeed = vspeed * (missing == 0) + missing * NaN_filler
missing = (elevations == nan_e)
if num.sometrue(missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_e.nc'
raise_(DataMissingValuesError, msg)
else:
elevations = elevations * (missing == 0) + missing * NaN_filler
number_of_times = times.shape[0]
number_of_latitudes = latitudes.shape[0]
number_of_longitudes = longitudes.shape[0]
assert amplitudes.shape[0] == number_of_times
assert amplitudes.shape[1] == number_of_latitudes
assert amplitudes.shape[2] == number_of_longitudes
if verbose:
_show_stats((latitudes, longitudes), times, amplitudes, \
(uspeed, vspeed), elevations)
# print number_of_latitudes, number_of_longitudes
number_of_points = number_of_latitudes * number_of_longitudes
number_of_volumes = (number_of_latitudes - 1) * (number_of_longitudes -
1) * 2
file_h.close()
file_u.close()
file_v.close()
file_e.close()
# NetCDF file definition
outfile = NetCDFFile(swwname, netcdf_mode_w)
description = 'Converted from Ferret files: %s, %s, %s, %s' \
% (basename_in + '_ha.nc',
basename_in + '_ua.nc',
basename_in + '_va.nc',
basename_in + '_e.nc')
# Create new file
starttime = times[0]
sww = Write_sww(['elevation'], ['stage', 'xmomentum', 'ymomentum'])
sww.store_header(outfile,
times,
number_of_volumes,
number_of_points,
description=description,
verbose=verbose,
sww_precision=netcdf_float)
# Store
from anuga.coordinate_transforms.redfearn import redfearn
x = num.zeros(number_of_points, num.float) #Easting
y = num.zeros(number_of_points, num.float) #Northing
if verbose:
log.critical('Making triangular grid')
# Check zone boundaries
refzone, _, _ = redfearn(latitudes[0], longitudes[0])
vertices = {}
i = 0
for k, lat in enumerate(latitudes): # Y direction
for l, lon in enumerate(longitudes): # X direction
vertices[l, k] = i
_, easting, northing = redfearn(lat, lon)
#msg = 'Zone boundary crossed at longitude =', lon
#assert zone == refzone, msg
#print '%7.2f %7.2f %8.2f %8.2f' %(lon, lat, easting, northing)
x[i] = easting
y[i] = northing
i += 1
#Construct 2 triangles per 'rectangular' element
volumes = []
for l in range(number_of_longitudes - 1): # X direction
for k in range(number_of_latitudes - 1): # Y direction
v1 = vertices[l, k + 1]
v2 = vertices[l, k]
v3 = vertices[l + 1, k + 1]
v4 = vertices[l + 1, k]
volumes.append([v1, v2, v3]) #Upper element
volumes.append([v4, v3, v2]) #Lower element
volumes = num.array(volumes, num.int) #array default#
if origin is None:
origin = Geo_reference(refzone, min(x), min(y))
geo_ref = write_NetCDF_georeference(origin, outfile)
if elevation is not None:
z = elevation
else:
if inverted_bathymetry:
z = -1 * elevations
else:
z = elevations
#FIXME: z should be obtained from MOST and passed in here
#FIXME use the Write_sww instance(sww) to write this info
z = num.resize(z, outfile.variables['elevation'][:].shape)
outfile.variables['x'][:] = x - geo_ref.get_xllcorner()
outfile.variables['y'][:] = y - geo_ref.get_yllcorner()
#outfile.variables['z'][:] = z #FIXME HACK for bacwards compat.
outfile.variables['elevation'][:] = z
outfile.variables['volumes'][:] = volumes.astype(
num.int32) #For Opteron 64
#Time stepping
stage = outfile.variables['stage']
xmomentum = outfile.variables['xmomentum']
ymomentum = outfile.variables['ymomentum']
if verbose:
log.critical('Converting quantities')
n = len(times)
for j in range(n):
if verbose and j % (old_div((n + 10), 10)) == 0:
log.critical(' Doing %d of %d' % (j, n))
i = 0
for k in range(number_of_latitudes): # Y direction
for l in range(number_of_longitudes): # X direction
w = old_div(zscale * amplitudes[j, k, l], 100) + mean_stage
stage[j, i] = w
h = w - z[i]
xmomentum[j, i] = old_div(uspeed[j, k, l], 100) * h
ymomentum[j, i] = old_div(vspeed[j, k, l], 100) * h
i += 1
#outfile.close()
#FIXME: Refactor using code from file_function.statistics
#Something like print swwstats(swwname)
if verbose:
time_info = times, starttime, mint, maxt
_show_sww_stats(outfile, swwname, geo_ref, time_info)
outfile.close()
