def test_triangulation_points_georeference(self):
#
#
filename = tempfile.mktemp("_data_manager.sww")
outfile = NetCDFFile(filename, netcdf_mode_w)
points_utm = num.array([[0.,0.],[1.,1.], [0.,1.]])
volumes = [[0,1,2]]
elevation = [0,1,2]
new_origin = None
points_georeference = Geo_reference(56, 1, 554354)
points_utm = points_georeference.change_points_geo_ref(points_utm)
times = [0, 10]
number_of_volumes = len(volumes)
number_of_points = len(points_utm)
sww = Write_sww(['elevation'], ['stage', 'xmomentum', 'ymomentum'])
sww.store_header(outfile, times, number_of_volumes,
number_of_points, description='fully sick testing',
verbose=self.verbose,sww_precision=netcdf_float)
sww.store_triangulation(outfile, points_utm, volumes,
elevation, new_origin=new_origin,
points_georeference=points_georeference,
verbose=self.verbose)
outfile.close()
fid = NetCDFFile(filename)
x = fid.variables['x'][:]
y = fid.variables['y'][:]
results_georef = Geo_reference()
results_georef.read_NetCDF(fid)
assert results_georef == points_georeference
fid.close()
assert num.allclose(num.array(map(None, x,y)), points_utm)
os.remove(filename)
def test_triangulationII(self):
#
#
filename = tempfile.mktemp("_data_manager.sww")
outfile = NetCDFFile(filename, netcdf_mode_w)
points_utm = num.array([[0., 0.], [1., 1.], [0., 1.]])
volumes = [[0, 1, 2]]
elevation = [0, 1, 2]
new_origin = None
#new_origin = Geo_reference(56, 0, 0)
times = [0, 10]
number_of_volumes = len(volumes)
number_of_points = len(points_utm)
sww = Write_sww(['elevation'], ['stage', 'xmomentum', 'ymomentum'])
sww.store_header(outfile,
times,
number_of_volumes,
number_of_points,
description='fully sick testing',
verbose=self.verbose,
sww_precision=netcdf_float)
sww.store_triangulation(outfile,
points_utm,
volumes,
new_origin=new_origin,
verbose=self.verbose)
sww.store_static_quantities(outfile, elevation=elevation)
outfile.close()
fid = NetCDFFile(filename)
x = fid.variables['x'][:]
y = fid.variables['y'][:]
results_georef = Geo_reference()
results_georef.read_NetCDF(fid)
assert results_georef == Geo_reference(zone=None,
xllcorner=0,
yllcorner=0)
fid.close()
assert num.allclose(num.array(list(zip(x, y))), points_utm)
os.remove(filename)
def test_triangulation_2_geo_refs(self):
#
#
filename = tempfile.mktemp("_data_manager.sww")
outfile = NetCDFFile(filename, netcdf_mode_w)
points_utm = num.array([[0., 0.], [1., 1.], [0., 1.]])
volumes = [[0, 1, 2]]
elevation = [0, 1, 2]
new_origin = Geo_reference(56, 1, 1)
points_georeference = Geo_reference(56, 0, 0)
points_utm = points_georeference.change_points_geo_ref(points_utm)
times = [0, 10]
number_of_volumes = len(volumes)
number_of_points = len(points_utm)
sww = Write_sww(['elevation'], ['stage', 'xmomentum', 'ymomentum'])
sww.store_header(outfile,
times,
number_of_volumes,
number_of_points,
description='fully sick testing',
verbose=self.verbose,
sww_precision=netcdf_float)
sww.store_triangulation(outfile,
points_utm,
volumes,
elevation,
new_origin=new_origin,
points_georeference=points_georeference,
verbose=self.verbose)
outfile.close()
fid = NetCDFFile(filename)
x = fid.variables['x'][:]
y = fid.variables['y'][:]
results_georef = Geo_reference()
results_georef.read_NetCDF(fid)
assert results_georef == new_origin
fid.close()
absolute = Geo_reference(56, 0, 0)
assert num.allclose(
num.array(
absolute.change_points_geo_ref(map(None, x, y), new_origin)),
points_utm)
os.remove(filename)
def test_triangulationII(self):
#
#
DEFAULT_ZONE = 0 # Not documented anywhere what this should be.
filename = tempfile.mktemp("_data_manager.sww")
outfile = NetCDFFile(filename, netcdf_mode_w)
points_utm = num.array([[0.,0.],[1.,1.], [0.,1.]])
volumes = [[0,1,2]]
elevation = [0,1,2]
new_origin = None
#new_origin = Geo_reference(56, 0, 0)
times = [0, 10]
number_of_volumes = len(volumes)
number_of_points = len(points_utm)
sww = Write_sww(['elevation'], ['stage', 'xmomentum', 'ymomentum'])
sww.store_header(outfile, times, number_of_volumes,
number_of_points, description='fully sick testing',
verbose=self.verbose,sww_precision=netcdf_float)
sww.store_triangulation(outfile, points_utm, volumes,
new_origin=new_origin,
verbose=self.verbose)
sww.store_static_quantities(outfile, elevation=elevation)
outfile.close()
fid = NetCDFFile(filename)
x = fid.variables['x'][:]
y = fid.variables['y'][:]
results_georef = Geo_reference()
results_georef.read_NetCDF(fid)
assert results_georef == Geo_reference(DEFAULT_ZONE, 0, 0)
fid.close()
assert num.allclose(num.array(map(None, x,y)), points_utm)
os.remove(filename)
def create_SWW_input(lista_puntos = 'lista_puntos.txt', time_series = 'time_series.txt',
sww_file_out = 'boundary_fixed.sww', num_params = 3, read_dataframe = False):
if read_dataframe:
df = pd.read_csv(time_series, sep = '\t', index_col= 0)
else:
tipo_datos = np.float32()
#-------------------------------------------------
#Lee archivos con posiciones y series de tiempo
header = 3
fid = open(lista_puntos,'r')
line = fid.readline()
fields = line.split()
nx = int(fields[1])
line = fid.readline()
fields = line.split()
ny = int(fields[1])
line = fid.readline()
fields = line.split()
dt = float(fields[1])
fid.close()
points_utm = np.loadtxt(lista_puntos, dtype = tipo_datos, skiprows = header)[:,0:2]
number_of_points = len(points_utm)
elevation = np.loadtxt(lista_puntos, dtype = tipo_datos, skiprows = header)[:,2]
#lee el tamaño del tiempo, y lo hace relativo a 0
dummy = np.loadtxt(time_series, dtype = tipo_datos, skiprows = header, delimiter = '\t',
usecols = (1,2))
number_of_times = len(dummy)
times = np.arange(0,number_of_times*dt,dt)
#determina número de datos válidos
fid = open(time_series,'r')
line = fid.readline()
line = fid.readline()
fields = line.split()
num_datos = (len(fields)-1)/(3*num_params)
fid.close()
index = []
for i in range(1, num_datos*3, 3):
index.append(int(fields[i][:-1]))
index_tupla = tuple(np.array(index)-1)
stage_aux = np.loadtxt(time_series, dtype = tipo_datos, skiprows = header, delimiter = '\t',
usecols = tuple(range(1,num_datos+1)))
xmom_aux = np.loadtxt(time_series, dtype = tipo_datos, skiprows = header, delimiter = '\t',
usecols = tuple(range(num_datos+1,2*num_datos+1)))
ymom_aux = np.loadtxt(time_series, dtype = tipo_datos, skiprows = header, delimiter = '\t',
usecols = tuple(range(2*num_datos+1,3*num_datos+1)))
#crea
stage = np.zeros((number_of_times,number_of_points),dtype = tipo_datos)
xmom = np.zeros((number_of_times,number_of_points),dtype = tipo_datos)
ymom = np.zeros((number_of_times,number_of_points),dtype = tipo_datos)
stage[:,index_tupla] = stage_aux
xmom[:,index_tupla] = xmom_aux
ymom[:,index_tupla] = ymom_aux
#triangulacion a mano
vertices = np.zeros((nx, ny))
i = 0
for k in range(ny):
for l in range(nx):
vertices[l, k] = i
i += 1
volumes = []
for l in range(nx-1):
for k in range(ny-1):
v1 = vertices[l, k+1]
v2 = vertices[l, k]
v3 = vertices[l+1, k+1]
v4 = vertices[l+1, k]
volumes.append([v1, v2, v3])
volumes.append([v4, v3, v2])
volumes = np.array(volumes, np.int)
##Escribe archivo SWW
# outfile = NetCDFFile(sww_file_out, netcdf_mode_w)
outfile = netcdf.Dataset(sww_file_out, netcdf_mode_w)
new_origin = Geo_reference(-1, 0, 0)
number_of_volumes = len(volumes)
sww = Write_sww(['elevation'], ['stage', 'xmomentum', 'ymomentum'])
sww.store_header(outfile, times, number_of_volumes, number_of_points,
description='SWW creado en PRDW',
sww_precision = netcdf_float,
verbose=True)
sww.store_triangulation(outfile, points_utm, volumes,
new_origin = new_origin,
verbose = True)
sww.store_static_quantities(outfile, elevation = elevation, verbose = True)
for i in range(len(times)):
sww.store_quantities(outfile, sww_precision = netcdf_float, slice_index = i,
verbose = True,
stage = stage[i,:],
xmomentum = xmom[i,:],
ymomentum = ymom[i,:])
outfile.close()
def _sww_merge_parallel_non_smooth(swwfiles, output, verbose=False, delete_old=False):
"""
Merge a list of sww files into a single file.
Used to merge files created by parallel runs.
The sww files to be merged must have exactly the same timesteps.
It is assumed that the separate sww files have been stored in non_smooth
format.
Note that some advanced information and custom quantities may not be
exported.
swwfiles is a list of .sww files to merge.
output is the output filename, including .sww extension.
verbose True to log output information
"""
if verbose:
print "MERGING SWW Files"
first_file = True
tri_offset = 0
for filename in swwfiles:
if verbose:
print 'Reading file ', filename, ':'
fid = NetCDFFile(filename, netcdf_mode_r)
if first_file:
times = fid.variables['time'][:]
n_steps = len(times)
number_of_timesteps = fid.dimensions['number_of_timesteps']
#print n_steps, number_of_timesteps
starttime = int(fid.starttime)
out_s_quantities = {}
out_d_quantities = {}
out_s_c_quantities = {}
out_d_c_quantities = {}
xllcorner = fid.xllcorner
yllcorner = fid.yllcorner
number_of_global_triangles = int(fid.number_of_global_triangles)
number_of_global_nodes = int(fid.number_of_global_nodes)
number_of_global_triangle_vertices = 3*number_of_global_triangles
order = fid.order
xllcorner = fid.xllcorner;
yllcorner = fid.yllcorner ;
zone = fid.zone;
false_easting = fid.false_easting;
false_northing = fid.false_northing;
datum = fid.datum;
projection = fid.projection;
g_volumes = num.arange(number_of_global_triangles*3).reshape(-1,3)
g_x = num.zeros((number_of_global_triangle_vertices,),num.float32)
g_y = num.zeros((number_of_global_triangle_vertices,),num.float32)
g_points = num.zeros((number_of_global_triangle_vertices,2),num.float32)
#=======================================
# Deal with the vertex based variables
#=======================================
quantities = set(['elevation', 'friction', 'stage', 'xmomentum',
'ymomentum', 'xvelocity', 'yvelocity', 'height'])
variables = set(fid.variables.keys())
quantities = list(quantities & variables)
static_quantities = []
dynamic_quantities = []
for quantity in quantities:
# Test if elevation is static
if n_steps == fid.variables[quantity].shape[0]:
dynamic_quantities.append(quantity)
else:
static_quantities.append(quantity)
# Static Quantities are stored as a 1D array
for quantity in static_quantities:
out_s_quantities[quantity] = num.zeros((3*number_of_global_triangles,),num.float32)
#=======================================
# Deal with the centroid based variables
#=======================================
quantities = set(['elevation_c', 'friction_c', 'stage_c', 'xmomentum_c',
'ymomentum_c', 'xvelocity_c', 'yvelocity_c', 'height_c'])
variables = set(fid.variables.keys())
quantities = list(quantities & variables)
static_c_quantities = []
dynamic_c_quantities = []
for quantity in quantities:
# Test if quantity is static
if n_steps == fid.variables[quantity].shape[0]:
dynamic_c_quantities.append(quantity)
else:
static_c_quantities.append(quantity)
for quantity in static_c_quantities:
out_s_c_quantities[quantity] = num.zeros((number_of_global_triangles,),num.float32)
description = 'merged:' + getattr(fid, 'description')
first_file = False
# Read in from files and add to global arrays
tri_l2g = fid.variables['tri_l2g'][:]
node_l2g = fid.variables['node_l2g'][:]
tri_full_flag = fid.variables['tri_full_flag'][:]
f_ids = num.argwhere(tri_full_flag==1).reshape(-1,)
f_gids = tri_l2g[f_ids]
g_vids = (3*f_gids.reshape(-1,1) + num.array([0,1,2])).reshape(-1,)
l_vids = (3*f_ids.reshape(-1,1) + num.array([0,1,2])).reshape(-1,)
l_x = num.array(fid.variables['x'][:],dtype=num.float32)
l_y = num.array(fid.variables['y'][:],dtype=num.float32)
g_x[g_vids] = l_x[l_vids]
g_y[g_vids] = l_y[l_vids]
g_points[g_vids,0] = g_x[g_vids]
g_points[g_vids,1] = g_y[g_vids]
## Read in static quantities
for quantity in static_quantities:
q = fid.variables[quantity]
out_s_quantities[quantity][g_vids] = \
num.array(q).astype(num.float32)[l_vids]
#num.array(q,dtype=num.float32)[l_vids]
# Read in static c quantities
for quantity in static_c_quantities:
q = fid.variables[quantity]
out_s_c_quantities[quantity][f_gids] = \
num.array(q).astype(num.float32)[f_ids]
#num.array(q,dtype=num.float32)[f_ids]
fid.close()
#---------------------------
# Write out the SWW file
#---------------------------
if verbose:
print 'Writing file ', output, ':'
fido = NetCDFFile(output, netcdf_mode_w)
sww = Write_sww(static_quantities, dynamic_quantities, static_c_quantities, dynamic_c_quantities)
sww.store_header(fido, starttime,
number_of_global_triangles,
number_of_global_triangles*3,
description=description,
sww_precision=netcdf_float32)
from anuga.coordinate_transforms.geo_reference import Geo_reference
geo_reference = Geo_reference()
sww.store_triangulation(fido, g_points, g_volumes, points_georeference=geo_reference)
fido.order = order
fido.xllcorner = xllcorner;
fido.yllcorner = yllcorner ;
fido.zone = zone;
fido.false_easting = false_easting;
fido.false_northing = false_northing;
fido.datum = datum;
fido.projection = projection;
sww.store_static_quantities(fido, verbose=verbose, **out_s_quantities)
sww.store_static_quantities_centroid(fido, verbose=verbose, **out_s_c_quantities)
# Write out all the dynamic quantities for each timestep
for i in range(n_steps):
fido.variables['time'][i] = times[i]
for q in (dynamic_quantities + dynamic_c_quantities):
if verbose:
print ' Writing quantity: ',q
# Initialise q_values with zeros
if q in dynamic_quantities:
q_values = num.zeros((n_steps, 3*number_of_global_triangles), num.float32)
elif q in dynamic_c_quantities:
q_values = num.zeros((n_steps, number_of_global_triangles), num.float32)
# Read the quantities one at a time, to reduce memory usage
for filename in swwfiles:
fid = NetCDFFile(filename, netcdf_mode_r)
# Index information
tri_l2g = fid.variables['tri_l2g'][:]
node_l2g = fid.variables['node_l2g'][:]
tri_full_flag = fid.variables['tri_full_flag'][:]
f_ids = num.argwhere(tri_full_flag==1).reshape(-1,)
f_gids = tri_l2g[f_ids]
g_vids = (3*f_gids.reshape(-1,1) + num.array([0,1,2])).reshape(-1,)
l_vids = (3*f_ids.reshape(-1,1) + num.array([0,1,2])).reshape(-1,)
for i in range(n_steps):
# Different indices for vertex and centroid quantities
if q in dynamic_quantities:
q_values[i][g_vids] = \
num.array(fid.variables[q][i], dtype=num.float32)[l_vids]
elif q in dynamic_c_quantities:
q_values[i][f_gids] = \
num.array(fid.variables[q][i], dtype=num.float32)[f_ids]
fid.close()
# Write to the file
for i in range(n_steps):
fido.variables[q][i] = q_values[i]
if q in dynamic_quantities:
# This updates the _range values
q_range = fido.variables[q + Write_sww.RANGE][:]
q_values_min = num.min(q_values)
if q_values_min < q_range[0]:
fido.variables[q + Write_sww.RANGE][0] = q_values_min
q_values_max = num.max(q_values)
if q_values_max > q_range[1]:
fido.variables[q + Write_sww.RANGE][1] = q_values_max
fido.close()
if delete_old:
import os
for filename in swwfiles:
if verbose:
print 'Deleting file ', filename, ':'
os.remove(filename)
def _sww_merge(swwfiles, output, verbose=False):
"""
Merge a list of sww files into a single file.
May be useful for parallel runs. Note that colinear points and
edges are not merged: there will essentially be multiple meshes within
the one sww file.
The sww files to be merged must have exactly the same timesteps. Note
that some advanced information and custom quantities may not be
exported.
swwfiles is a list of .sww files to merge.
output is the output filename, including .sww extension.
verbose True to log output information
"""
if verbose:
print "MERGING SWW Files"
static_quantities = ['elevation']
dynamic_quantities = ['stage', 'xmomentum', 'ymomentum']
first_file = True
tri_offset = 0
for filename in swwfiles:
if verbose:
print 'Reading file ', filename, ':'
fid = NetCDFFile(filename, netcdf_mode_r)
tris = fid.variables['volumes'][:]
if first_file:
times = fid.variables['time'][:]
x = []
y = []
out_tris = list(tris)
out_s_quantities = {}
out_d_quantities = {}
xllcorner = fid.xllcorner
yllcorner = fid.yllcorner
order = fid.order
xllcorner = fid.xllcorner;
yllcorner = fid.yllcorner ;
zone = fid.zone;
false_easting = fid.false_easting;
false_northing = fid.false_northing;
datum = fid.datum;
projection = fid.projection;
for quantity in static_quantities:
out_s_quantities[quantity] = []
# Quantities are stored as a 2D array of timesteps x data.
for quantity in dynamic_quantities:
out_d_quantities[quantity] = [ [] for _ in range(len(times))]
description = 'merged:' + getattr(fid, 'description')
first_file = False
else:
for tri in tris:
# Advance new tri indices to point at newly appended points.
verts = [vertex+tri_offset for vertex in tri]
out_tris.append(verts)
try: # works with netcdf4
num_pts = len(fid.dimensions['number_of_points'])
except: # works with scientific.io.netcdf
num_pts = int(fid.dimensions['number_of_points'])
tri_offset += num_pts
if verbose:
print ' new triangle index offset is ', tri_offset
x.extend(list(fid.variables['x'][:]))
y.extend(list(fid.variables['y'][:]))
# Grow the list of static quantities associated with the x,y points
for quantity in static_quantities:
out_s_quantities[quantity].extend(fid.variables[quantity][:])
#Collate all dynamic quantities according to their timestep
for quantity in dynamic_quantities:
time_chunks = fid.variables[quantity][:]
for i, time_chunk in enumerate(time_chunks):
out_d_quantities[quantity][i].extend(time_chunk)
# Mash all points into a single big list
points = [[xx, yy] for xx, yy in zip(x, y)]
points = num.asarray(points).astype(netcdf_float32)
fid.close()
#---------------------------
# Write out the SWW file
#---------------------------
if verbose:
print 'Writing file ', output, ':'
fido = NetCDFFile(output, netcdf_mode_w)
sww = Write_sww(static_quantities, dynamic_quantities)
sww.store_header(fido, times,
len(out_tris),
len(points),
description=description,
sww_precision=netcdf_float32)
from anuga.coordinate_transforms.geo_reference import Geo_reference
geo_reference = Geo_reference()
sww.store_triangulation(fido, points, out_tris, points_georeference=geo_reference)
fido.order = order
fido.xllcorner = xllcorner;
fido.yllcorner = yllcorner ;
fido.zone = zone;
fido.false_easting = false_easting;
fido.false_northing = false_northing;
fido.datum = datum;
fido.projection = projection;
sww.store_static_quantities(fido, verbose=verbose, **out_s_quantities)
# Write out all the dynamic quantities for each timestep
for q in dynamic_quantities:
q_values = out_d_quantities[q]
for i, time_slice in enumerate(q_values):
fido.variables[q][i] = num.array(time_slice, netcdf_float32)
# This updates the _range values
q_range = fido.variables[q + Write_sww.RANGE][:]
q_values_min = num.min(q_values)
if q_values_min < q_range[0]:
fido.variables[q + Write_sww.RANGE][0] = q_values_min
q_values_max = num.max(q_values)
if q_values_max > q_range[1]:
fido.variables[q + Write_sww.RANGE][1] = q_values_max
fido.close()
def _sww_merge_parallel_smooth(swwfiles, output, verbose=False, delete_old=False):
"""
Merge a list of sww files into a single file.
Use to merge files created by parallel runs.
The sww files to be merged must have exactly the same timesteps.
It is assumed that the separate sww files have been stored in non_smooth
format.
Note that some advanced information and custom quantities may not be
exported.
swwfiles is a list of .sww files to merge.
output is the output filename, including .sww extension.
verbose True to log output information
"""
if verbose:
print "MERGING SWW Files"
first_file = True
tri_offset = 0
for filename in swwfiles:
if verbose:
print 'Reading file ', filename, ':'
fid = NetCDFFile(filename, netcdf_mode_r)
if first_file:
times = fid.variables['time'][:]
n_steps = len(times)
#number_of_timesteps = fid.dimensions['number_of_timesteps']
#print n_steps, number_of_timesteps
starttime = int(fid.starttime)
out_s_quantities = {}
out_d_quantities = {}
out_s_c_quantities = {}
out_d_c_quantities = {}
xllcorner = fid.xllcorner
yllcorner = fid.yllcorner
number_of_global_triangles = int(fid.number_of_global_triangles)
number_of_global_nodes = int(fid.number_of_global_nodes)
order = fid.order
xllcorner = fid.xllcorner;
yllcorner = fid.yllcorner ;
zone = fid.zone;
false_easting = fid.false_easting;
false_northing = fid.false_northing;
datum = fid.datum;
projection = fid.projection;
g_volumes = num.zeros((number_of_global_triangles,3),num.int)
g_x = num.zeros((number_of_global_nodes,),num.float32)
g_y = num.zeros((number_of_global_nodes,),num.float32)
g_points = num.zeros((number_of_global_nodes,2),num.float32)
#=====================================
# Deal with the vertex based variables
#=====================================
quantities = set(['elevation', 'friction', 'stage', 'xmomentum',
'ymomentum', 'xvelocity', 'yvelocity', 'height'])
variables = set(fid.variables.keys())
quantities = list(quantities & variables)
static_quantities = []
dynamic_quantities = []
for quantity in quantities:
# Test if quantity is static
if n_steps == fid.variables[quantity].shape[0]:
dynamic_quantities.append(quantity)
else:
static_quantities.append(quantity)
for quantity in static_quantities:
out_s_quantities[quantity] = num.zeros((number_of_global_nodes,),num.float32)
# Quantities are stored as a 2D array of timesteps x data.
for quantity in dynamic_quantities:
out_d_quantities[quantity] = \
num.zeros((n_steps,number_of_global_nodes),num.float32)
#=======================================
# Deal with the centroid based variables
#=======================================
quantities = set(['elevation_c', 'friction_c', 'stage_c', 'xmomentum_c',
'ymomentum_c', 'xvelocity_c', 'yvelocity_c', 'height_c'])
variables = set(fid.variables.keys())
quantities = list(quantities & variables)
static_c_quantities = []
dynamic_c_quantities = []
for quantity in quantities:
# Test if quantity is static
if n_steps == fid.variables[quantity].shape[0]:
dynamic_c_quantities.append(quantity)
else:
static_c_quantities.append(quantity)
for quantity in static_c_quantities:
out_s_c_quantities[quantity] = num.zeros((number_of_global_triangles,),num.float32)
# Quantities are stored as a 2D array of timesteps x data.
for quantity in dynamic_c_quantities:
out_d_c_quantities[quantity] = \
num.zeros((n_steps,number_of_global_triangles),num.float32)
description = 'merged:' + getattr(fid, 'description')
first_file = False
# Read in from files and add to global arrays
tri_l2g = fid.variables['tri_l2g'][:]
node_l2g = fid.variables['node_l2g'][:]
tri_full_flag = fid.variables['tri_full_flag'][:]
volumes = num.array(fid.variables['volumes'][:],dtype=num.int)
l_volumes = num.zeros_like(volumes)
l_old_volumes = num.zeros_like(volumes)
# Change the local node ids to global id in the
# volume array
# FIXME SR: Surely we can knock up a numpy way of doing this
#for i in range(len(l_volumes)):
# g_n0 = node_l2g[volumes[i,0]]
# g_n1 = node_l2g[volumes[i,1]]
# g_n2 = node_l2g[volumes[i,2]]
#
# l_old_volumes[i,:] = [g_n0,g_n1,g_n2]
g_n0 = node_l2g[volumes[:,0]].reshape(-1,1)
g_n1 = node_l2g[volumes[:,1]].reshape(-1,1)
g_n2 = node_l2g[volumes[:,2]].reshape(-1,1)
#print g_n0.shape
l_volumes = num.hstack((g_n0,g_n1,g_n2))
#assert num.allclose(l_volumes, l_old_volumes)
# Just pick out the full triangles
ftri_ids = num.where(tri_full_flag>0)
ftri_l2g = num.compress(tri_full_flag, tri_l2g)
#f_ids = num.argwhere(tri_full_flag==1).reshape(-1,)
#f_gids = tri_l2g[f_ids]
#print l_volumes
#print tri_full_flag
#print tri_l2g
#print ftri_l2g
f_volumes0 = num.compress(tri_full_flag,volumes[:,0])
f_volumes1 = num.compress(tri_full_flag,volumes[:,1])
f_volumes2 = num.compress(tri_full_flag,volumes[:,2])
g_volumes[ftri_l2g,0] = node_l2g[f_volumes0]
g_volumes[ftri_l2g,1] = node_l2g[f_volumes1]
g_volumes[ftri_l2g,2] = node_l2g[f_volumes2]
#fg_volumes = num.compress(tri_full_flag,l_volumes,axis=0)
#g_volumes[ftri_l2g] = fg_volumes
#g_x[node_l2g] = fid.variables['x']
#g_y[node_l2g] = fid.variables['y']
g_points[node_l2g,0] = fid.variables['x'][:]
g_points[node_l2g,1] = fid.variables['y'][:]
#print number_of_timesteps
# FIXME SR: It seems that some of the "ghost" node quantity values
# are being storded. We should only store those nodes which are associated with
# full triangles. So we need an index array of "full" nodes, ie those in
# full triangles
#use numpy.compress and numpy.unique to get "full nodes
f_volumes = num.compress(tri_full_flag,volumes,axis=0)
fl_nodes = num.unique(f_volumes)
f_node_l2g = node_l2g[fl_nodes]
#print len(node_l2g)
#print len(fl_nodes)
# Read in static quantities
for quantity in static_quantities:
#out_s_quantities[quantity][node_l2g] = \
# num.array(fid.variables[quantity],dtype=num.float32)
q = fid.variables[quantity]
#print quantity, q.shape
out_s_quantities[quantity][f_node_l2g] = \
num.array(q[:],dtype=num.float32)[fl_nodes]
#Collate all dynamic quantities according to their timestep
for quantity in dynamic_quantities:
q = fid.variables[quantity]
#print q.shape
for i in range(n_steps):
#out_d_quantities[quantity][i][node_l2g] = \
# num.array(q[i],dtype=num.float32)
out_d_quantities[quantity][i][f_node_l2g] = \
num.array(q[i],dtype=num.float32)[fl_nodes]
# Read in static c quantities
for quantity in static_c_quantities:
#out_s_quantities[quantity][node_l2g] = \
# num.array(fid.variables[quantity],dtype=num.float32)
q = fid.variables[quantity]
out_s_c_quantities[quantity][ftri_l2g] = \
num.array(q).astype(num.float32)[ftri_ids]
#Collate all dynamic c quantities according to their timestep
for quantity in dynamic_c_quantities:
q = fid.variables[quantity]
#print q.shape
for i in range(n_steps):
out_d_c_quantities[quantity][i][ftri_l2g] = \
num.array(q[i]).astype(num.float32)[ftri_ids]
fid.close()
#---------------------------
# Write out the SWW file
#---------------------------
#print g_points.shape
#print number_of_global_triangles
#print number_of_global_nodes
if verbose:
print 'Writing file ', output, ':'
fido = NetCDFFile(output, netcdf_mode_w)
sww = Write_sww(static_quantities, dynamic_quantities, static_c_quantities, dynamic_c_quantities)
sww.store_header(fido, starttime,
number_of_global_triangles,
number_of_global_nodes,
description=description,
sww_precision=netcdf_float32)
from anuga.coordinate_transforms.geo_reference import Geo_reference
geo_reference = Geo_reference()
sww.store_triangulation(fido, g_points, g_volumes, points_georeference=geo_reference)
fido.order = order
fido.xllcorner = xllcorner;
fido.yllcorner = yllcorner ;
fido.zone = zone;
fido.false_easting = false_easting;
fido.false_northing = false_northing;
fido.datum = datum;
fido.projection = projection;
sww.store_static_quantities(fido, verbose=verbose, **out_s_quantities)
sww.store_static_quantities_centroid(fido, verbose=verbose, **out_s_c_quantities)
# Write out all the dynamic quantities for each timestep
for i in range(n_steps):
fido.variables['time'][i] = times[i]
for q in dynamic_quantities:
q_values = out_d_quantities[q]
if verbose:
print ' Writing quantity: ',q
for i in range(n_steps):
fido.variables[q][i] = q_values[i]
# This updates the _range values
q_range = fido.variables[q + Write_sww.RANGE][:]
q_values_min = num.min(q_values)
if q_values_min < q_range[0]:
fido.variables[q + Write_sww.RANGE][0] = q_values_min
q_values_max = num.max(q_values)
if q_values_max > q_range[1]:
fido.variables[q + Write_sww.RANGE][1] = q_values_max
for q in dynamic_c_quantities:
if verbose:
print ' Writing quantity: ',q
q_values = out_d_c_quantities[q]
for i in range(n_steps):
fido.variables[q][i] = q_values[i]
#print out_s_quantities
#print out_d_quantities
#print g_x
#print g_y
#print g_volumes
fido.close()
if delete_old:
import os
for filename in swwfiles:
if verbose:
print 'Deleting file ', filename, ':'
os.remove(filename)
def ferret2sww(basename_in, name_out=None,
verbose=False,
minlat=None, maxlat=None,
minlon=None, maxlon=None,
mint=None, maxt=None, mean_stage=0,
origin=None, zscale=1,
fail_on_NaN=True,
NaN_filler=0,
elevation=None,
inverted_bathymetry=True
): #FIXME: Bathymetry should be obtained
#from MOST somehow.
#Alternatively from elsewhere
#or, as a last resort,
#specified here.
#The value of -100 will work
#for the Wollongong tsunami
#scenario but is very hacky
"""Convert MOST and 'Ferret' NetCDF format for wave propagation to
sww format native to abstract_2d_finite_volumes.
Specify only basename_in and read files of the form
basefilename_ha.nc, basefilename_ua.nc, basefilename_va.nc containing
relative height, x-velocity and y-velocity, respectively.
Also convert latitude and longitude to UTM. All coordinates are
assumed to be given in the GDA94 datum.
min's and max's: If omitted - full extend is used.
To include a value min may equal it, while max must exceed it.
Lat and lon are assuemd to be in decimal degrees
origin is a 3-tuple with geo referenced
UTM coordinates (zone, easting, northing)
nc format has values organised as HA[TIME, LATITUDE, LONGITUDE]
which means that longitude is the fastest
varying dimension (row major order, so to speak)
ferret2sww uses grid points as vertices in a triangular grid
counting vertices from lower left corner upwards, then right
"""
from anuga.file.netcdf import NetCDFFile
_assert_lat_long(minlat, maxlat, minlon, maxlon)
if name_out != None and name_out[-4:] != '.sww':
raise IOError('Output file %s should be of type .sww.' % name_out)
# Get NetCDF data
if verbose: log.critical('Reading files %s_*.nc' % basename_in)
# Wave amplitude (cm)
file_h = NetCDFFile(basename_in + '_ha.nc', netcdf_mode_r)
# Velocity (x) (cm/s)
file_u = NetCDFFile(basename_in + '_ua.nc', netcdf_mode_r)
# Velocity (y) (cm/s)
file_v = NetCDFFile(basename_in + '_va.nc', netcdf_mode_r)
# Elevation (z) (m)
file_e = NetCDFFile(basename_in + '_e.nc', netcdf_mode_r)
if name_out is None:
swwname = basename_in + '.sww'
else:
swwname = name_out
# Get dimensions of file_h
for dimension in file_h.dimensions.keys():
if dimension[:3] == 'LON':
dim_h_longitude = dimension
if dimension[:3] == 'LAT':
dim_h_latitude = dimension
if dimension[:4] == 'TIME':
dim_h_time = dimension
times = file_h.variables[dim_h_time]
latitudes = file_h.variables[dim_h_latitude]
longitudes = file_h.variables[dim_h_longitude]
kmin, kmax, lmin, lmax = get_min_max_indices(latitudes[:],
longitudes[:],
minlat, maxlat,
minlon, maxlon)
# get dimensions for file_e
for dimension in file_e.dimensions.keys():
if dimension[:3] == 'LON':
dim_e_longitude = dimension
if dimension[:3] == 'LAT':
dim_e_latitude = dimension
# get dimensions for file_u
for dimension in file_u.dimensions.keys():
if dimension[:3] == 'LON':
dim_u_longitude = dimension
if dimension[:3] == 'LAT':
dim_u_latitude = dimension
# get dimensions for file_v
for dimension in file_v.dimensions.keys():
if dimension[:3] == 'LON':
dim_v_longitude = dimension
if dimension[:3] == 'LAT':
dim_v_latitude = dimension
# Precision used by most for lat/lon is 4 or 5 decimals
e_lat = num.around(file_e.variables[dim_e_latitude][:], 5)
e_lon = num.around(file_e.variables[dim_e_longitude][:], 5)
# Check that files are compatible
assert num.allclose(latitudes, file_u.variables[dim_u_latitude])
assert num.allclose(latitudes, file_v.variables[dim_v_latitude])
assert num.allclose(latitudes, e_lat)
assert num.allclose(longitudes, file_u.variables[dim_u_longitude])
assert num.allclose(longitudes, file_v.variables[dim_v_longitude])
assert num.allclose(longitudes, e_lon)
if mint is None:
jmin = 0
mint = times[0]
else:
jmin = num.searchsorted(times, mint)
# numpy.int32 didn't work in slicing of amplitude below
jmin = int(jmin)
if maxt is None:
jmax = len(times)
maxt = times[-1]
else:
jmax = num.searchsorted(times, maxt)
# numpy.int32 didn't work in slicing of amplitude below
jmax = int(jmax)
kmin, kmax, lmin, lmax = get_min_max_indices(latitudes[:],
longitudes[:],
minlat, maxlat,
minlon, maxlon)
times = times[jmin:jmax]
latitudes = latitudes[kmin:kmax]
longitudes = longitudes[lmin:lmax]
if verbose: log.critical('cropping')
zname = 'ELEVATION'
amplitudes = file_h.variables['HA'][jmin:jmax, kmin:kmax, lmin:lmax]
uspeed = file_u.variables['UA'][jmin:jmax, kmin:kmax, lmin:lmax] #Lon
vspeed = file_v.variables['VA'][jmin:jmax, kmin:kmax, lmin:lmax] #Lat
elevations = file_e.variables[zname][kmin:kmax, lmin:lmax]
# Get missing values
nan_ha = file_h.variables['HA'].missing_value
nan_ua = file_u.variables['UA'].missing_value
nan_va = file_v.variables['VA'].missing_value
if hasattr(file_e.variables[zname],'missing_value'):
nan_e = file_e.variables[zname].missing_value
else:
nan_e = None
# Cleanup
missing = (amplitudes == nan_ha)
if num.sometrue (missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_ha.nc'
raise DataMissingValuesError, msg
else:
amplitudes = amplitudes*(missing==0) + missing*NaN_filler
missing = (uspeed == nan_ua)
if num.sometrue (missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_ua.nc'
raise DataMissingValuesError, msg
else:
uspeed = uspeed*(missing==0) + missing*NaN_filler
missing = (vspeed == nan_va)
if num.sometrue (missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_va.nc'
raise DataMissingValuesError, msg
else:
vspeed = vspeed*(missing==0) + missing*NaN_filler
missing = (elevations == nan_e)
if num.sometrue (missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_e.nc'
raise DataMissingValuesError, msg
else:
elevations = elevations*(missing==0) + missing*NaN_filler
number_of_times = times.shape[0]
number_of_latitudes = latitudes.shape[0]
number_of_longitudes = longitudes.shape[0]
assert amplitudes.shape[0] == number_of_times
assert amplitudes.shape[1] == number_of_latitudes
assert amplitudes.shape[2] == number_of_longitudes
if verbose:
_show_stats((latitudes, longitudes), times, amplitudes, \
(uspeed, vspeed), elevations)
# print number_of_latitudes, number_of_longitudes
number_of_points = number_of_latitudes * number_of_longitudes
number_of_volumes = (number_of_latitudes-1) * (number_of_longitudes-1) * 2
file_h.close()
file_u.close()
file_v.close()
file_e.close()
# NetCDF file definition
outfile = NetCDFFile(swwname, netcdf_mode_w)
description = 'Converted from Ferret files: %s, %s, %s, %s' \
% (basename_in + '_ha.nc',
basename_in + '_ua.nc',
basename_in + '_va.nc',
basename_in + '_e.nc')
# Create new file
starttime = times[0]
sww = Write_sww(['elevation'], ['stage', 'xmomentum', 'ymomentum'])
sww.store_header(outfile, times, number_of_volumes,
number_of_points, description=description,
verbose=verbose, sww_precision=netcdf_float)
# Store
from anuga.coordinate_transforms.redfearn import redfearn
x = num.zeros(number_of_points, num.float) #Easting
y = num.zeros(number_of_points, num.float) #Northing
if verbose:
log.critical('Making triangular grid')
# Check zone boundaries
refzone, _, _ = redfearn(latitudes[0], longitudes[0])
vertices = {}
i = 0
for k, lat in enumerate(latitudes): # Y direction
for l, lon in enumerate(longitudes): # X direction
vertices[l, k] = i
_, easting, northing = redfearn(lat, lon)
#msg = 'Zone boundary crossed at longitude =', lon
#assert zone == refzone, msg
#print '%7.2f %7.2f %8.2f %8.2f' %(lon, lat, easting, northing)
x[i] = easting
y[i] = northing
i += 1
#Construct 2 triangles per 'rectangular' element
volumes = []
for l in range(number_of_longitudes-1): # X direction
for k in range(number_of_latitudes-1): # Y direction
v1 = vertices[l, k+1]
v2 = vertices[l, k]
v3 = vertices[l+1, k+1]
v4 = vertices[l+1, k]
volumes.append([v1, v2, v3]) #Upper element
volumes.append([v4, v3, v2]) #Lower element
volumes = num.array(volumes, num.int) #array default#
if origin is None:
origin = Geo_reference(refzone, min(x), min(y))
geo_ref = write_NetCDF_georeference(origin, outfile)
if elevation is not None:
z = elevation
else:
if inverted_bathymetry:
z = -1 * elevations
else:
z = elevations
#FIXME: z should be obtained from MOST and passed in here
#FIXME use the Write_sww instance(sww) to write this info
z = num.resize(z, outfile.variables['elevation'][:].shape)
outfile.variables['x'][:] = x - geo_ref.get_xllcorner()
outfile.variables['y'][:] = y - geo_ref.get_yllcorner()
#outfile.variables['z'][:] = z #FIXME HACK for bacwards compat.
outfile.variables['elevation'][:] = z
outfile.variables['volumes'][:] = volumes.astype(num.int32) #For Opteron 64
#Time stepping
stage = outfile.variables['stage']
xmomentum = outfile.variables['xmomentum']
ymomentum = outfile.variables['ymomentum']
if verbose:
log.critical('Converting quantities')
n = len(times)
for j in range(n):
if verbose and j % ((n+10)/10) == 0:
log.critical(' Doing %d of %d' % (j, n))
i = 0
for k in range(number_of_latitudes): # Y direction
for l in range(number_of_longitudes): # X direction
w = zscale * amplitudes[j, k, l] / 100 + mean_stage
stage[j, i] = w
h = w - z[i]
xmomentum[j, i] = uspeed[j, k, l]/100*h
ymomentum[j, i] = vspeed[j, k, l]/100*h
i += 1
#outfile.close()
#FIXME: Refactor using code from file_function.statistics
#Something like print swwstats(swwname)
if verbose:
time_info = times, starttime, mint, maxt
_show_sww_stats(outfile, swwname, geo_ref, time_info)
outfile.close()
def ferret2sww(
basename_in,
name_out=None,
verbose=False,
minlat=None,
maxlat=None,
minlon=None,
maxlon=None,
mint=None,
maxt=None,
mean_stage=0,
origin=None,
zscale=1,
fail_on_NaN=True,
NaN_filler=0,
elevation=None,
inverted_bathymetry=True): #FIXME: Bathymetry should be obtained
#from MOST somehow.
#Alternatively from elsewhere
#or, as a last resort,
#specified here.
#The value of -100 will work
#for the Wollongong tsunami
#scenario but is very hacky
"""Convert MOST and 'Ferret' NetCDF format for wave propagation to
sww format native to abstract_2d_finite_volumes.
Specify only basename_in and read files of the form
basefilename_ha.nc, basefilename_ua.nc, basefilename_va.nc containing
relative height, x-velocity and y-velocity, respectively.
Also convert latitude and longitude to UTM. All coordinates are
assumed to be given in the GDA94 datum.
min's and max's: If omitted - full extend is used.
To include a value min may equal it, while max must exceed it.
Lat and lon are assuemd to be in decimal degrees
origin is a 3-tuple with geo referenced
UTM coordinates (zone, easting, northing)
nc format has values organised as HA[TIME, LATITUDE, LONGITUDE]
which means that longitude is the fastest
varying dimension (row major order, so to speak)
ferret2sww uses grid points as vertices in a triangular grid
counting vertices from lower left corner upwards, then right
"""
from anuga.file.netcdf import NetCDFFile
_assert_lat_long(minlat, maxlat, minlon, maxlon)
if name_out != None and name_out[-4:] != '.sww':
raise IOError('Output file %s should be of type .sww.' % name_out)
# Get NetCDF data
if verbose: log.critical('Reading files %s_*.nc' % basename_in)
# Wave amplitude (cm)
file_h = NetCDFFile(basename_in + '_ha.nc', netcdf_mode_r)
# Velocity (x) (cm/s)
file_u = NetCDFFile(basename_in + '_ua.nc', netcdf_mode_r)
# Velocity (y) (cm/s)
file_v = NetCDFFile(basename_in + '_va.nc', netcdf_mode_r)
# Elevation (z) (m)
file_e = NetCDFFile(basename_in + '_e.nc', netcdf_mode_r)
if name_out is None:
swwname = basename_in + '.sww'
else:
swwname = name_out
# Get dimensions of file_h
for dimension in list(file_h.dimensions.keys()):
if dimension[:3] == 'LON':
dim_h_longitude = dimension
if dimension[:3] == 'LAT':
dim_h_latitude = dimension
if dimension[:4] == 'TIME':
dim_h_time = dimension
times = file_h.variables[dim_h_time]
latitudes = file_h.variables[dim_h_latitude]
longitudes = file_h.variables[dim_h_longitude]
kmin, kmax, lmin, lmax = get_min_max_indices(latitudes[:], longitudes[:],
minlat, maxlat, minlon,
maxlon)
# get dimensions for file_e
for dimension in list(file_e.dimensions.keys()):
if dimension[:3] == 'LON':
dim_e_longitude = dimension
if dimension[:3] == 'LAT':
dim_e_latitude = dimension
# get dimensions for file_u
for dimension in list(file_u.dimensions.keys()):
if dimension[:3] == 'LON':
dim_u_longitude = dimension
if dimension[:3] == 'LAT':
dim_u_latitude = dimension
# get dimensions for file_v
for dimension in list(file_v.dimensions.keys()):
if dimension[:3] == 'LON':
dim_v_longitude = dimension
if dimension[:3] == 'LAT':
dim_v_latitude = dimension
# Precision used by most for lat/lon is 4 or 5 decimals
e_lat = num.around(file_e.variables[dim_e_latitude][:], 5)
e_lon = num.around(file_e.variables[dim_e_longitude][:], 5)
# Check that files are compatible
assert num.allclose(latitudes, file_u.variables[dim_u_latitude])
assert num.allclose(latitudes, file_v.variables[dim_v_latitude])
assert num.allclose(latitudes, e_lat)
assert num.allclose(longitudes, file_u.variables[dim_u_longitude])
assert num.allclose(longitudes, file_v.variables[dim_v_longitude])
assert num.allclose(longitudes, e_lon)
if mint is None:
jmin = 0
mint = times[0]
else:
jmin = num.searchsorted(times, mint)
# numpy.int32 didn't work in slicing of amplitude below
jmin = int(jmin)
if maxt is None:
jmax = len(times)
maxt = times[-1]
else:
jmax = num.searchsorted(times, maxt)
# numpy.int32 didn't work in slicing of amplitude below
jmax = int(jmax)
kmin, kmax, lmin, lmax = get_min_max_indices(latitudes[:], longitudes[:],
minlat, maxlat, minlon,
maxlon)
times = times[jmin:jmax]
latitudes = latitudes[kmin:kmax]
longitudes = longitudes[lmin:lmax]
if verbose: log.critical('cropping')
zname = 'ELEVATION'
amplitudes = file_h.variables['HA'][jmin:jmax, kmin:kmax, lmin:lmax]
uspeed = file_u.variables['UA'][jmin:jmax, kmin:kmax, lmin:lmax] #Lon
vspeed = file_v.variables['VA'][jmin:jmax, kmin:kmax, lmin:lmax] #Lat
elevations = file_e.variables[zname][kmin:kmax, lmin:lmax]
# Get missing values
nan_ha = file_h.variables['HA'].missing_value
nan_ua = file_u.variables['UA'].missing_value
nan_va = file_v.variables['VA'].missing_value
if hasattr(file_e.variables[zname], 'missing_value'):
nan_e = file_e.variables[zname].missing_value
else:
nan_e = None
# Cleanup
missing = (amplitudes == nan_ha)
if num.sometrue(missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_ha.nc'
raise_(DataMissingValuesError, msg)
else:
amplitudes = amplitudes * (missing == 0) + missing * NaN_filler
missing = (uspeed == nan_ua)
if num.sometrue(missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_ua.nc'
raise_(DataMissingValuesError, msg)
else:
uspeed = uspeed * (missing == 0) + missing * NaN_filler
missing = (vspeed == nan_va)
if num.sometrue(missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_va.nc'
raise_(DataMissingValuesError, msg)
else:
vspeed = vspeed * (missing == 0) + missing * NaN_filler
missing = (elevations == nan_e)
if num.sometrue(missing):
if fail_on_NaN:
msg = 'NetCDFFile %s contains missing values' \
% basename_in + '_e.nc'
raise_(DataMissingValuesError, msg)
else:
elevations = elevations * (missing == 0) + missing * NaN_filler
number_of_times = times.shape[0]
number_of_latitudes = latitudes.shape[0]
number_of_longitudes = longitudes.shape[0]
assert amplitudes.shape[0] == number_of_times
assert amplitudes.shape[1] == number_of_latitudes
assert amplitudes.shape[2] == number_of_longitudes
if verbose:
_show_stats((latitudes, longitudes), times, amplitudes, \
(uspeed, vspeed), elevations)
# print number_of_latitudes, number_of_longitudes
number_of_points = number_of_latitudes * number_of_longitudes
number_of_volumes = (number_of_latitudes - 1) * (number_of_longitudes -
1) * 2
file_h.close()
file_u.close()
file_v.close()
file_e.close()
# NetCDF file definition
outfile = NetCDFFile(swwname, netcdf_mode_w)
description = 'Converted from Ferret files: %s, %s, %s, %s' \
% (basename_in + '_ha.nc',
basename_in + '_ua.nc',
basename_in + '_va.nc',
basename_in + '_e.nc')
# Create new file
starttime = times[0]
sww = Write_sww(['elevation'], ['stage', 'xmomentum', 'ymomentum'])
sww.store_header(outfile,
times,
number_of_volumes,
number_of_points,
description=description,
verbose=verbose,
sww_precision=netcdf_float)
# Store
from anuga.coordinate_transforms.redfearn import redfearn
x = num.zeros(number_of_points, num.float) #Easting
y = num.zeros(number_of_points, num.float) #Northing
if verbose:
log.critical('Making triangular grid')
# Check zone boundaries
refzone, _, _ = redfearn(latitudes[0], longitudes[0])
vertices = {}
i = 0
for k, lat in enumerate(latitudes): # Y direction
for l, lon in enumerate(longitudes): # X direction
vertices[l, k] = i
_, easting, northing = redfearn(lat, lon)
#msg = 'Zone boundary crossed at longitude =', lon
#assert zone == refzone, msg
#print '%7.2f %7.2f %8.2f %8.2f' %(lon, lat, easting, northing)
x[i] = easting
y[i] = northing
i += 1
#Construct 2 triangles per 'rectangular' element
volumes = []
for l in range(number_of_longitudes - 1): # X direction
for k in range(number_of_latitudes - 1): # Y direction
v1 = vertices[l, k + 1]
v2 = vertices[l, k]
v3 = vertices[l + 1, k + 1]
v4 = vertices[l + 1, k]
volumes.append([v1, v2, v3]) #Upper element
volumes.append([v4, v3, v2]) #Lower element
volumes = num.array(volumes, num.int) #array default#
if origin is None:
origin = Geo_reference(refzone, min(x), min(y))
geo_ref = write_NetCDF_georeference(origin, outfile)
if elevation is not None:
z = elevation
else:
if inverted_bathymetry:
z = -1 * elevations
else:
z = elevations
#FIXME: z should be obtained from MOST and passed in here
#FIXME use the Write_sww instance(sww) to write this info
z = num.resize(z, outfile.variables['elevation'][:].shape)
outfile.variables['x'][:] = x - geo_ref.get_xllcorner()
outfile.variables['y'][:] = y - geo_ref.get_yllcorner()
#outfile.variables['z'][:] = z #FIXME HACK for bacwards compat.
outfile.variables['elevation'][:] = z
outfile.variables['volumes'][:] = volumes.astype(
num.int32) #For Opteron 64
#Time stepping
stage = outfile.variables['stage']
xmomentum = outfile.variables['xmomentum']
ymomentum = outfile.variables['ymomentum']
if verbose:
log.critical('Converting quantities')
n = len(times)
for j in range(n):
if verbose and j % (old_div((n + 10), 10)) == 0:
log.critical(' Doing %d of %d' % (j, n))
i = 0
for k in range(number_of_latitudes): # Y direction
for l in range(number_of_longitudes): # X direction
w = old_div(zscale * amplitudes[j, k, l], 100) + mean_stage
stage[j, i] = w
h = w - z[i]
xmomentum[j, i] = old_div(uspeed[j, k, l], 100) * h
ymomentum[j, i] = old_div(vspeed[j, k, l], 100) * h
i += 1
#outfile.close()
#FIXME: Refactor using code from file_function.statistics
#Something like print swwstats(swwname)
if verbose:
time_info = times, starttime, mint, maxt
_show_sww_stats(outfile, swwname, geo_ref, time_info)
outfile.close()
def sts2sww_mesh(basename_in, basename_out=None,
spatial_thinning=1, verbose=False):
from anuga.mesh_engine.mesh_engine import NoTrianglesError
from anuga.pmesh.mesh import Mesh
if verbose:
print "Starting sts2sww_mesh"
mean_stage=0.
zscale=1.
if (basename_in[:-4]=='.sts'):
stsname = basename_in
else:
stsname = basename_in + '.sts'
if verbose: print "Reading sts NetCDF file: %s" %stsname
infile = NetCDFFile(stsname, netcdf_mode_r)
cellsize = infile.cellsize
ncols = infile.ncols
nrows = infile.nrows
no_data = infile.no_data
refzone = infile.zone
x_origin = infile.xllcorner
y_origin = infile.yllcorner
origin = num.array([x_origin, y_origin])
x = infile.variables['x'][:]
y = infile.variables['y'][:]
times = infile.variables['time'][:]
wind_speed_full = infile.variables['wind_speed'][:]
wind_angle_full = infile.variables['wind_angle'][:]
pressure_full = infile.variables['barometric_pressure'][:]
infile.close()
number_of_points = nrows*ncols
points_utm = num.zeros((number_of_points,2),num.float)
points_utm[:,0]=x+x_origin
points_utm[:,1]=y+y_origin
thinned_indices=[]
for i in range(number_of_points):
if (i/ncols==0 or i/ncols==ncols-1 or (i/ncols)%(spatial_thinning)==0):
if ( i%(spatial_thinning)==0 or i%nrows==0 or i%nrows==nrows-1 ):
thinned_indices.append(i)
#Spatial thinning
points_utm=points_utm[thinned_indices]
number_of_points = points_utm.shape[0]
number_of_timesteps = wind_speed_full.shape[0]
wind_speed = num.empty((number_of_timesteps,number_of_points),dtype=float)
wind_angle = num.empty((number_of_timesteps,number_of_points),dtype=float)
barometric_pressure = num.empty((number_of_timesteps,number_of_points),dtype=float)
if verbose:
print "Total number of points: ", nrows*ncols
print "Number of thinned points: ", number_of_points
for i in xrange(number_of_timesteps):
wind_speed[i] = wind_speed_full[i,thinned_indices]
wind_angle[i] = wind_angle_full[i,thinned_indices]
barometric_pressure[i] = pressure_full[i,thinned_indices]
#P.plot(points_utm[:,0],points_utm[:,1],'ro')
#P.show()
if verbose:
print "Generating sww triangulation of gems data"
mesh = Mesh()
mesh.add_vertices(points_utm)
mesh.auto_segment(smooth_indents=True, expand_pinch=True)
mesh.auto_segment(mesh.shape.get_alpha() * 1.1)
try:
mesh.generate_mesh(minimum_triangle_angle=0.0, verbose=False)
except NoTrianglesError:
# This is a bit of a hack, going in and changing the data structure.
mesh.holes = []
mesh.generate_mesh(minimum_triangle_angle=0.0, verbose=False)
mesh_dic = mesh.Mesh2MeshList()
points_utm=ensure_numeric(points_utm)
assert num.alltrue(ensure_numeric(mesh_dic['generatedpointlist'])
== ensure_numeric(points_utm))
volumes = mesh_dic['generatedtrianglelist']
# Write sww intro and grid stuff.
if (basename_out is not None and basename_out[:-4]=='.sww'):
swwname = basename_out
else:
swwname = basename_in + '.sww'
if verbose: 'Output to %s' % swwname
if verbose:
print "Writing sww wind and pressure field file"
outfile = NetCDFFile(swwname, netcdf_mode_w)
sww = Write_sww([], ['wind_speed','wind_angle','barometric_pressure'])
sww.store_header(outfile, times, len(volumes), len(points_utm),
verbose=verbose, sww_precision='d')
outfile.mean_stage = mean_stage
outfile.zscale = zscale
sww.store_triangulation(outfile, points_utm, volumes,
refzone,
new_origin=origin, #check effect of this line
verbose=verbose)
if verbose:
print 'Converting quantities'
# Read in a time slice from the sts file and write it to the SWW file
#print wind_angle[0,:10]
for i in range(len(times)):
sww.store_quantities(outfile,
slice_index=i,
verbose=verbose,
wind_speed=wind_speed[i,:],
wind_angle=wind_angle[i,:],
barometric_pressure=barometric_pressure[i,:],
sww_precision=num.float)
if verbose:
sww.verbose_quantities(outfile)
outfile.close()
def urs_ungridded2sww(basename_in='o', basename_out=None, verbose=False,
mint=None, maxt=None,
mean_stage=0,
origin=None,
hole_points_UTM=None,
zscale=1):
"""
Convert URS C binary format for wave propagation to
sww format native to abstract_2d_finite_volumes.
Specify only basename_in and read files of the form
basefilename-z-mux, basefilename-e-mux and
basefilename-n-mux containing relative height,
x-velocity and y-velocity, respectively.
Also convert latitude and longitude to UTM. All coordinates are
assumed to be given in the GDA94 datum. The latitude and longitude
information is assumed ungridded grid.
min's and max's: If omitted - full extend is used.
To include a value min ans max may equal it.
Lat and lon are assumed to be in decimal degrees.
origin is a 3-tuple with geo referenced
UTM coordinates (zone, easting, northing)
It will be the origin of the sww file. This shouldn't be used,
since all of anuga should be able to handle an arbitary origin.
The mux point info is NOT relative to this origin.
URS C binary format has data organised as TIME, LONGITUDE, LATITUDE
which means that latitude is the fastest
varying dimension (row major order, so to speak)
In URS C binary the latitudes and longitudes are in assending order.
Note, interpolations of the resulting sww file will be different
from results of urs2sww. This is due to the interpolation
function used, and the different grid structure between urs2sww
and this function.
Interpolating data that has an underlying gridded source can
easily end up with different values, depending on the underlying
mesh.
consider these 4 points
50 -50
0 0
The grid can be
-
|\| A
-
or;
-
|/| B
-
If a point is just below the center of the midpoint, it will have a
+ve value in grid A and a -ve value in grid B.
"""
from anuga.mesh_engine.mesh_engine import NoTrianglesError
from anuga.pmesh.mesh import Mesh
files_in = [basename_in + WAVEHEIGHT_MUX_LABEL,
basename_in + EAST_VELOCITY_LABEL,
basename_in + NORTH_VELOCITY_LABEL]
quantities = ['HA','UA','VA']
# instantiate urs_points of the three mux files.
mux = {}
for quantity, file in map(None, quantities, files_in):
mux[quantity] = Read_urs(file)
# Could check that the depth is the same. (hashing)
# handle to a mux file to do depth stuff
a_mux = mux[quantities[0]]
# Convert to utm
lat = a_mux.lonlatdep[:,1]
long = a_mux.lonlatdep[:,0]
points_utm, zone = convert_from_latlon_to_utm(latitudes=lat,
longitudes=long)
elevation = a_mux.lonlatdep[:,2] * -1
# grid (create a mesh from the selected points)
# This mesh has a problem. Triangles are streched over ungridded areas.
# If these areas could be described as holes in pmesh, that would be great.
# I can't just get the user to selection a point in the middle.
# A boundary is needed around these points.
# But if the zone of points is obvious enough auto-segment should do
# a good boundary.
mesh = Mesh()
mesh.add_vertices(points_utm)
mesh.auto_segment(smooth_indents=True, expand_pinch=True)
# To try and avoid alpha shape 'hugging' too much
mesh.auto_segment(mesh.shape.get_alpha() * 1.1)
if hole_points_UTM is not None:
point = ensure_absolute(hole_points_UTM)
mesh.add_hole(point[0], point[1])
try:
mesh.generate_mesh(minimum_triangle_angle=0.0, verbose=False)
except NoTrianglesError:
# This is a bit of a hack, going in and changing the data structure.
mesh.holes = []
mesh.generate_mesh(minimum_triangle_angle=0.0, verbose=False)
mesh_dic = mesh.Mesh2MeshList()
#mesh.export_mesh_file(basename_in + '_168.tsh')
#import sys; sys.exit()
# These are the times of the mux file
mux_times = []
for i in range(a_mux.time_step_count):
mux_times.append(a_mux.time_step * i)
(mux_times_start_i, mux_times_fin_i) = read_time_from_mux(mux_times, mint, maxt)
times = mux_times[mux_times_start_i:mux_times_fin_i]
if mux_times_start_i == mux_times_fin_i:
# Close the mux files
for quantity, file in map(None, quantities, files_in):
mux[quantity].close()
msg = "Due to mint and maxt there's no time info in the boundary SWW."
raise Exception(msg)
# If this raise is removed there is currently no downstream errors
points_utm=ensure_numeric(points_utm)
assert num.alltrue(ensure_numeric(mesh_dic['generatedpointlist'])
== ensure_numeric(points_utm))
volumes = mesh_dic['generatedtrianglelist']
# Write sww intro and grid stuff.
if basename_out is None:
swwname = basename_in + '.sww'
else:
swwname = basename_out + '.sww'
if verbose: log.critical('Output to %s' % swwname)
outfile = NetCDFFile(swwname, netcdf_mode_w)
# For a different way of doing this, check out tsh2sww
# work out sww_times and the index range this covers
sww = Write_sww(['elevation'], ['stage', 'xmomentum', 'ymomentum'])
sww.store_header(outfile, times, len(volumes), len(points_utm),
verbose=verbose, sww_precision=netcdf_float)
outfile.mean_stage = mean_stage
outfile.zscale = zscale
sww.store_triangulation(outfile, points_utm, volumes,
zone,
new_origin=origin,
verbose=verbose)
sww.store_static_quantities(outfile, elevation=elevation)
if verbose: log.critical('Converting quantities')
# Read in a time slice from each mux file and write it to the SWW file
j = 0
for ha, ua, va in map(None, mux['HA'], mux['UA'], mux['VA']):
if j >= mux_times_start_i and j < mux_times_fin_i:
stage = zscale*ha + mean_stage
h = stage - elevation
xmomentum = ua*h
ymomentum = -1 * va * h # -1 since in mux files south is positive.
sww.store_quantities(outfile,
slice_index=j-mux_times_start_i,
verbose=verbose,
stage=stage,
xmomentum=xmomentum,
ymomentum=ymomentum,
sww_precision=num.float)
j += 1
if verbose: sww.verbose_quantities(outfile)
outfile.close()
def _sww_merge_parallel_non_smooth(swwfiles,
output,
verbose=False,
delete_old=False):
"""
Merge a list of sww files into a single file.
Used to merge files created by parallel runs.
The sww files to be merged must have exactly the same timesteps.
It is assumed that the separate sww files have been stored in non_smooth
format.
Note that some advanced information and custom quantities may not be
exported.
swwfiles is a list of .sww files to merge.
output is the output filename, including .sww extension.
verbose True to log output information
"""
if verbose:
print "MERGING SWW Files"
first_file = True
tri_offset = 0
for filename in swwfiles:
if verbose:
print 'Reading file ', filename, ':'
fid = NetCDFFile(filename, netcdf_mode_r)
if first_file:
times = fid.variables['time'][:]
n_steps = len(times)
number_of_timesteps = fid.dimensions['number_of_timesteps']
#print n_steps, number_of_timesteps
starttime = int(fid.starttime)
out_s_quantities = {}
out_d_quantities = {}
out_s_c_quantities = {}
out_d_c_quantities = {}
xllcorner = fid.xllcorner
yllcorner = fid.yllcorner
number_of_global_triangles = int(fid.number_of_global_triangles)
number_of_global_nodes = int(fid.number_of_global_nodes)
number_of_global_triangle_vertices = 3 * number_of_global_triangles
order = fid.order
xllcorner = fid.xllcorner
yllcorner = fid.yllcorner
zone = fid.zone
false_easting = fid.false_easting
false_northing = fid.false_northing
datum = fid.datum
projection = fid.projection
g_volumes = num.arange(number_of_global_triangles * 3).reshape(
-1, 3)
g_x = num.zeros((number_of_global_triangle_vertices, ),
num.float32)
g_y = num.zeros((number_of_global_triangle_vertices, ),
num.float32)
g_points = num.zeros((number_of_global_triangle_vertices, 2),
num.float32)
#=======================================
# Deal with the vertex based variables
#=======================================
quantities = set([
'elevation', 'friction', 'stage', 'xmomentum', 'ymomentum',
'xvelocity', 'yvelocity', 'height'
])
variables = set(fid.variables.keys())
quantities = list(quantities & variables)
static_quantities = []
dynamic_quantities = []
for quantity in quantities:
# Test if elevation is static
if n_steps == fid.variables[quantity].shape[0]:
dynamic_quantities.append(quantity)
else:
static_quantities.append(quantity)
# Static Quantities are stored as a 1D array
for quantity in static_quantities:
out_s_quantities[quantity] = num.zeros(
(3 * number_of_global_triangles, ), num.float32)
#=======================================
# Deal with the centroid based variables
#=======================================
quantities = set([
'elevation_c', 'friction_c', 'stage_c', 'xmomentum_c',
'ymomentum_c', 'xvelocity_c', 'yvelocity_c', 'height_c'
])
variables = set(fid.variables.keys())
quantities = list(quantities & variables)
static_c_quantities = []
dynamic_c_quantities = []
for quantity in quantities:
# Test if quantity is static
if n_steps == fid.variables[quantity].shape[0]:
dynamic_c_quantities.append(quantity)
else:
static_c_quantities.append(quantity)
for quantity in static_c_quantities:
out_s_c_quantities[quantity] = num.zeros(
(number_of_global_triangles, ), num.float32)
description = 'merged:' + getattr(fid, 'description')
first_file = False
# Read in from files and add to global arrays
tri_l2g = fid.variables['tri_l2g'][:]
node_l2g = fid.variables['node_l2g'][:]
tri_full_flag = fid.variables['tri_full_flag'][:]
f_ids = num.argwhere(tri_full_flag == 1).reshape(-1, )
f_gids = tri_l2g[f_ids]
g_vids = (3 * f_gids.reshape(-1, 1) + num.array([0, 1, 2])).reshape(
-1, )
l_vids = (3 * f_ids.reshape(-1, 1) + num.array([0, 1, 2])).reshape(
-1, )
l_x = num.array(fid.variables['x'][:], dtype=num.float32)
l_y = num.array(fid.variables['y'][:], dtype=num.float32)
g_x[g_vids] = l_x[l_vids]
g_y[g_vids] = l_y[l_vids]
g_points[g_vids, 0] = g_x[g_vids]
g_points[g_vids, 1] = g_y[g_vids]
## Read in static quantities
for quantity in static_quantities:
q = fid.variables[quantity]
out_s_quantities[quantity][g_vids] = \
num.array(q).astype(num.float32)[l_vids]
#num.array(q,dtype=num.float32)[l_vids]
# Read in static c quantities
for quantity in static_c_quantities:
q = fid.variables[quantity]
out_s_c_quantities[quantity][f_gids] = \
num.array(q).astype(num.float32)[f_ids]
#num.array(q,dtype=num.float32)[f_ids]
fid.close()
#---------------------------
# Write out the SWW file
#---------------------------
if verbose:
print 'Writing file ', output, ':'
fido = NetCDFFile(output, netcdf_mode_w)
sww = Write_sww(static_quantities, dynamic_quantities, static_c_quantities,
dynamic_c_quantities)
sww.store_header(fido,
starttime,
number_of_global_triangles,
number_of_global_triangles * 3,
description=description,
sww_precision=netcdf_float32)
from anuga.coordinate_transforms.geo_reference import Geo_reference
geo_reference = Geo_reference()
sww.store_triangulation(fido,
g_points,
g_volumes,
points_georeference=geo_reference)
fido.order = order
fido.xllcorner = xllcorner
fido.yllcorner = yllcorner
fido.zone = zone
fido.false_easting = false_easting
fido.false_northing = false_northing
fido.datum = datum
fido.projection = projection
sww.store_static_quantities(fido, verbose=verbose, **out_s_quantities)
sww.store_static_quantities_centroid(fido,
verbose=verbose,
**out_s_c_quantities)
# Write out all the dynamic quantities for each timestep
for i in range(n_steps):
fido.variables['time'][i] = times[i]
for q in (dynamic_quantities + dynamic_c_quantities):
# Initialise q_values with zeros
if q in dynamic_quantities:
q_values = num.zeros((n_steps, 3 * number_of_global_triangles),
num.float32)
elif q in dynamic_c_quantities:
q_values = num.zeros((n_steps, number_of_global_triangles),
num.float32)
# Read the quantities one at a time, to reduce memory usage
for filename in swwfiles:
fid = NetCDFFile(filename, netcdf_mode_r)
# Index information
tri_l2g = fid.variables['tri_l2g'][:]
node_l2g = fid.variables['node_l2g'][:]
tri_full_flag = fid.variables['tri_full_flag'][:]
f_ids = num.argwhere(tri_full_flag == 1).reshape(-1, )
f_gids = tri_l2g[f_ids]
g_vids = (3 * f_gids.reshape(-1, 1) +
num.array([0, 1, 2])).reshape(-1, )
l_vids = (3 * f_ids.reshape(-1, 1) + num.array([0, 1, 2])).reshape(
-1, )
for i in range(n_steps):
# Different indices for vertex and centroid quantities
if q in dynamic_quantities:
q_values[i][g_vids] = \
num.array(fid.variables[q][i], dtype=num.float32)[l_vids]
elif q in dynamic_c_quantities:
q_values[i][f_gids] = \
num.array(fid.variables[q][i], dtype=num.float32)[f_ids]
fid.close()
# Write to the file
for i in range(n_steps):
fido.variables[q][i] = q_values[i]
if q in dynamic_quantities:
# This updates the _range values
q_range = fido.variables[q + Write_sww.RANGE][:]
q_values_min = num.min(q_values)
if q_values_min < q_range[0]:
fido.variables[q + Write_sww.RANGE][0] = q_values_min
q_values_max = num.max(q_values)
if q_values_max > q_range[1]:
fido.variables[q + Write_sww.RANGE][1] = q_values_max
fido.close()
if delete_old:
import os
for filename in swwfiles:
if verbose:
print 'Deleting file ', filename, ':'
os.remove(filename)
def _sww_merge(swwfiles, output, verbose=False):
"""
Merge a list of sww files into a single file.
May be useful for parallel runs. Note that colinear points and
edges are not merged: there will essentially be multiple meshes within
the one sww file.
The sww files to be merged must have exactly the same timesteps. Note
that some advanced information and custom quantities may not be
exported.
swwfiles is a list of .sww files to merge.
output is the output filename, including .sww extension.
verbose True to log output information
"""
if verbose:
print "MERGING SWW Files"
static_quantities = ['elevation']
dynamic_quantities = ['stage', 'xmomentum', 'ymomentum']
first_file = True
tri_offset = 0
for filename in swwfiles:
if verbose:
print 'Reading file ', filename, ':'
fid = NetCDFFile(filename, netcdf_mode_r)
tris = fid.variables['volumes'][:]
if first_file:
times = fid.variables['time'][:]
x = []
y = []
out_tris = list(tris)
out_s_quantities = {}
out_d_quantities = {}
xllcorner = fid.xllcorner
yllcorner = fid.yllcorner
order = fid.order
xllcorner = fid.xllcorner
yllcorner = fid.yllcorner
zone = fid.zone
false_easting = fid.false_easting
false_northing = fid.false_northing
datum = fid.datum
projection = fid.projection
for quantity in static_quantities:
out_s_quantities[quantity] = []
# Quantities are stored as a 2D array of timesteps x data.
for quantity in dynamic_quantities:
out_d_quantities[quantity] = [[] for _ in range(len(times))]
description = 'merged:' + getattr(fid, 'description')
first_file = False
else:
for tri in tris:
# Advance new tri indices to point at newly appended points.
verts = [vertex + tri_offset for vertex in tri]
out_tris.append(verts)
try: # works with netcdf4
num_pts = len(fid.dimensions['number_of_points'])
except: # works with scientific.io.netcdf
num_pts = int(fid.dimensions['number_of_points'])
tri_offset += num_pts
if verbose:
print ' new triangle index offset is ', tri_offset
x.extend(list(fid.variables['x'][:]))
y.extend(list(fid.variables['y'][:]))
# Grow the list of static quantities associated with the x,y points
for quantity in static_quantities:
out_s_quantities[quantity].extend(fid.variables[quantity][:])
#Collate all dynamic quantities according to their timestep
for quantity in dynamic_quantities:
time_chunks = fid.variables[quantity][:]
for i, time_chunk in enumerate(time_chunks):
out_d_quantities[quantity][i].extend(time_chunk)
# Mash all points into a single big list
points = [[xx, yy] for xx, yy in zip(x, y)]
points = num.asarray(points).astype(netcdf_float32)
fid.close()
#---------------------------
# Write out the SWW file
#---------------------------
if verbose:
print 'Writing file ', output, ':'
fido = NetCDFFile(output, netcdf_mode_w)
sww = Write_sww(static_quantities, dynamic_quantities)
sww.store_header(fido,
times,
len(out_tris),
len(points),
description=description,
sww_precision=netcdf_float32)
from anuga.coordinate_transforms.geo_reference import Geo_reference
geo_reference = Geo_reference()
sww.store_triangulation(fido,
points,
out_tris,
points_georeference=geo_reference)
fido.order = order
fido.xllcorner = xllcorner
fido.yllcorner = yllcorner
fido.zone = zone
fido.false_easting = false_easting
fido.false_northing = false_northing
fido.datum = datum
fido.projection = projection
sww.store_static_quantities(fido, verbose=verbose, **out_s_quantities)
# Write out all the dynamic quantities for each timestep
for q in dynamic_quantities:
q_values = out_d_quantities[q]
for i, time_slice in enumerate(q_values):
fido.variables[q][i] = num.array(time_slice, netcdf_float32)
# This updates the _range values
q_range = fido.variables[q + Write_sww.RANGE][:]
q_values_min = num.min(q_values)
if q_values_min < q_range[0]:
fido.variables[q + Write_sww.RANGE][0] = q_values_min
q_values_max = num.max(q_values)
if q_values_max > q_range[1]:
fido.variables[q + Write_sww.RANGE][1] = q_values_max
fido.close()
def _sww_merge_parallel_smooth(swwfiles,
output,
verbose=False,
delete_old=False):
"""
Merge a list of sww files into a single file.
Use to merge files created by parallel runs.
The sww files to be merged must have exactly the same timesteps.
It is assumed that the separate sww files have been stored in non_smooth
format.
Note that some advanced information and custom quantities may not be
exported.
swwfiles is a list of .sww files to merge.
output is the output filename, including .sww extension.
verbose True to log output information
"""
if verbose:
print "MERGING SWW Files"
first_file = True
tri_offset = 0
for filename in swwfiles:
if verbose:
print 'Reading file ', filename, ':'
fid = NetCDFFile(filename, netcdf_mode_r)
if first_file:
times = fid.variables['time'][:]
n_steps = len(times)
#number_of_timesteps = fid.dimensions['number_of_timesteps']
#print n_steps, number_of_timesteps
starttime = int(fid.starttime)
out_s_quantities = {}
out_d_quantities = {}
out_s_c_quantities = {}
out_d_c_quantities = {}
xllcorner = fid.xllcorner
yllcorner = fid.yllcorner
number_of_global_triangles = int(fid.number_of_global_triangles)
number_of_global_nodes = int(fid.number_of_global_nodes)
order = fid.order
xllcorner = fid.xllcorner
yllcorner = fid.yllcorner
zone = fid.zone
false_easting = fid.false_easting
false_northing = fid.false_northing
datum = fid.datum
projection = fid.projection
g_volumes = num.zeros((number_of_global_triangles, 3), num.int)
g_x = num.zeros((number_of_global_nodes, ), num.float32)
g_y = num.zeros((number_of_global_nodes, ), num.float32)
g_points = num.zeros((number_of_global_nodes, 2), num.float32)
#=====================================
# Deal with the vertex based variables
#=====================================
quantities = set([
'elevation', 'friction', 'stage', 'xmomentum', 'ymomentum',
'xvelocity', 'yvelocity', 'height'
])
variables = set(fid.variables.keys())
quantities = list(quantities & variables)
static_quantities = []
dynamic_quantities = []
for quantity in quantities:
# Test if quantity is static
if n_steps == fid.variables[quantity].shape[0]:
dynamic_quantities.append(quantity)
else:
static_quantities.append(quantity)
for quantity in static_quantities:
out_s_quantities[quantity] = num.zeros(
(number_of_global_nodes, ), num.float32)
# Quantities are stored as a 2D array of timesteps x data.
for quantity in dynamic_quantities:
out_d_quantities[quantity] = \
num.zeros((n_steps,number_of_global_nodes),num.float32)
#=======================================
# Deal with the centroid based variables
#=======================================
quantities = set([
'elevation_c', 'friction_c', 'stage_c', 'xmomentum_c',
'ymomentum_c', 'xvelocity_c', 'yvelocity_c', 'height_c'
])
variables = set(fid.variables.keys())
quantities = list(quantities & variables)
static_c_quantities = []
dynamic_c_quantities = []
for quantity in quantities:
# Test if quantity is static
if n_steps == fid.variables[quantity].shape[0]:
dynamic_c_quantities.append(quantity)
else:
static_c_quantities.append(quantity)
for quantity in static_c_quantities:
out_s_c_quantities[quantity] = num.zeros(
(number_of_global_triangles, ), num.float32)
# Quantities are stored as a 2D array of timesteps x data.
for quantity in dynamic_c_quantities:
out_d_c_quantities[quantity] = \
num.zeros((n_steps,number_of_global_triangles),num.float32)
description = 'merged:' + getattr(fid, 'description')
first_file = False
# Read in from files and add to global arrays
tri_l2g = fid.variables['tri_l2g'][:]
node_l2g = fid.variables['node_l2g'][:]
tri_full_flag = fid.variables['tri_full_flag'][:]
volumes = num.array(fid.variables['volumes'][:], dtype=num.int)
l_volumes = num.zeros_like(volumes)
l_old_volumes = num.zeros_like(volumes)
# Change the local node ids to global id in the
# volume array
# FIXME SR: Surely we can knock up a numpy way of doing this
#for i in range(len(l_volumes)):
# g_n0 = node_l2g[volumes[i,0]]
# g_n1 = node_l2g[volumes[i,1]]
# g_n2 = node_l2g[volumes[i,2]]
#
# l_old_volumes[i,:] = [g_n0,g_n1,g_n2]
g_n0 = node_l2g[volumes[:, 0]].reshape(-1, 1)
g_n1 = node_l2g[volumes[:, 1]].reshape(-1, 1)
g_n2 = node_l2g[volumes[:, 2]].reshape(-1, 1)
#print g_n0.shape
l_volumes = num.hstack((g_n0, g_n1, g_n2))
#assert num.allclose(l_volumes, l_old_volumes)
# Just pick out the full triangles
ftri_ids = num.where(tri_full_flag > 0)
ftri_l2g = num.compress(tri_full_flag, tri_l2g)
#f_ids = num.argwhere(tri_full_flag==1).reshape(-1,)
#f_gids = tri_l2g[f_ids]
#print l_volumes
#print tri_full_flag
#print tri_l2g
#print ftri_l2g
f_volumes0 = num.compress(tri_full_flag, volumes[:, 0])
f_volumes1 = num.compress(tri_full_flag, volumes[:, 1])
f_volumes2 = num.compress(tri_full_flag, volumes[:, 2])
g_volumes[ftri_l2g, 0] = node_l2g[f_volumes0]
g_volumes[ftri_l2g, 1] = node_l2g[f_volumes1]
g_volumes[ftri_l2g, 2] = node_l2g[f_volumes2]
#fg_volumes = num.compress(tri_full_flag,l_volumes,axis=0)
#g_volumes[ftri_l2g] = fg_volumes
#g_x[node_l2g] = fid.variables['x']
#g_y[node_l2g] = fid.variables['y']
g_points[node_l2g, 0] = fid.variables['x'][:]
g_points[node_l2g, 1] = fid.variables['y'][:]
#print number_of_timesteps
# FIXME SR: It seems that some of the "ghost" node quantity values
# are being storded. We should only store those nodes which are associated with
# full triangles. So we need an index array of "full" nodes, ie those in
# full triangles
#use numpy.compress and numpy.unique to get "full nodes
f_volumes = num.compress(tri_full_flag, volumes, axis=0)
fl_nodes = num.unique(f_volumes)
f_node_l2g = node_l2g[fl_nodes]
#print len(node_l2g)
#print len(fl_nodes)
# Read in static quantities
for quantity in static_quantities:
#out_s_quantities[quantity][node_l2g] = \
# num.array(fid.variables[quantity],dtype=num.float32)
q = fid.variables[quantity]
#print quantity, q.shape
out_s_quantities[quantity][f_node_l2g] = \
num.array(q[:],dtype=num.float32)[fl_nodes]
#Collate all dynamic quantities according to their timestep
for quantity in dynamic_quantities:
q = fid.variables[quantity]
#print q.shape
for i in range(n_steps):
#out_d_quantities[quantity][i][node_l2g] = \
# num.array(q[i],dtype=num.float32)
out_d_quantities[quantity][i][f_node_l2g] = \
num.array(q[i],dtype=num.float32)[fl_nodes]
# Read in static c quantities
for quantity in static_c_quantities:
#out_s_quantities[quantity][node_l2g] = \
# num.array(fid.variables[quantity],dtype=num.float32)
q = fid.variables[quantity]
out_s_c_quantities[quantity][ftri_l2g] = \
num.array(q).astype(num.float32)[ftri_ids]
#Collate all dynamic c quantities according to their timestep
for quantity in dynamic_c_quantities:
q = fid.variables[quantity]
#print q.shape
for i in range(n_steps):
out_d_c_quantities[quantity][i][ftri_l2g] = \
num.array(q[i]).astype(num.float32)[ftri_ids]
fid.close()
#---------------------------
# Write out the SWW file
#---------------------------
#print g_points.shape
#print number_of_global_triangles
#print number_of_global_nodes
if verbose:
print 'Writing file ', output, ':'
fido = NetCDFFile(output, netcdf_mode_w)
sww = Write_sww(static_quantities, dynamic_quantities, static_c_quantities,
dynamic_c_quantities)
sww.store_header(fido,
starttime,
number_of_global_triangles,
number_of_global_nodes,
description=description,
sww_precision=netcdf_float32)
from anuga.coordinate_transforms.geo_reference import Geo_reference
geo_reference = Geo_reference()
sww.store_triangulation(fido,
g_points,
g_volumes,
points_georeference=geo_reference)
fido.order = order
fido.xllcorner = xllcorner
fido.yllcorner = yllcorner
fido.zone = zone
fido.false_easting = false_easting
fido.false_northing = false_northing
fido.datum = datum
fido.projection = projection
sww.store_static_quantities(fido, verbose=verbose, **out_s_quantities)
sww.store_static_quantities_centroid(fido,
verbose=verbose,
**out_s_c_quantities)
# Write out all the dynamic quantities for each timestep
for i in range(n_steps):
fido.variables['time'][i] = times[i]
for q in dynamic_quantities:
q_values = out_d_quantities[q]
for i in range(n_steps):
fido.variables[q][i] = q_values[i]
# This updates the _range values
q_range = fido.variables[q + Write_sww.RANGE][:]
q_values_min = num.min(q_values)
if q_values_min < q_range[0]:
fido.variables[q + Write_sww.RANGE][0] = q_values_min
q_values_max = num.max(q_values)
if q_values_max > q_range[1]:
fido.variables[q + Write_sww.RANGE][1] = q_values_max
for q in dynamic_c_quantities:
q_values = out_d_c_quantities[q]
for i in range(n_steps):
fido.variables[q][i] = q_values[i]
#print out_s_quantities
#print out_d_quantities
#print g_x
#print g_y
#print g_volumes
fido.close()
if delete_old:
import os
for filename in swwfiles:
if verbose:
print 'Deleting file ', filename, ':'
os.remove(filename)
def urs_ungridded2sww(basename_in='o',
basename_out=None,
verbose=False,
mint=None,
maxt=None,
mean_stage=0,
origin=None,
hole_points_UTM=None,
zscale=1):
"""
Convert URS C binary format for wave propagation to
sww format native to abstract_2d_finite_volumes.
Specify only basename_in and read files of the form
basefilename-z-mux, basefilename-e-mux and
basefilename-n-mux containing relative height,
x-velocity and y-velocity, respectively.
Also convert latitude and longitude to UTM. All coordinates are
assumed to be given in the GDA94 datum. The latitude and longitude
information is assumed ungridded grid.
min's and max's: If omitted - full extend is used.
To include a value min ans max may equal it.
Lat and lon are assumed to be in decimal degrees.
origin is a 3-tuple with geo referenced
UTM coordinates (zone, easting, northing)
It will be the origin of the sww file. This shouldn't be used,
since all of anuga should be able to handle an arbitary origin.
The mux point info is NOT relative to this origin.
URS C binary format has data organised as TIME, LONGITUDE, LATITUDE
which means that latitude is the fastest
varying dimension (row major order, so to speak)
In URS C binary the latitudes and longitudes are in assending order.
Note, interpolations of the resulting sww file will be different
from results of urs2sww. This is due to the interpolation
function used, and the different grid structure between urs2sww
and this function.
Interpolating data that has an underlying gridded source can
easily end up with different values, depending on the underlying
mesh.
consider these 4 points
50 -50
0 0
The grid can be
-
|\| A
-
or;
-
|/| B
-
If a point is just below the center of the midpoint, it will have a
+ve value in grid A and a -ve value in grid B.
"""
from anuga.mesh_engine.mesh_engine import NoTrianglesError
from anuga.pmesh.mesh import Mesh
files_in = [
basename_in + WAVEHEIGHT_MUX_LABEL, basename_in + EAST_VELOCITY_LABEL,
basename_in + NORTH_VELOCITY_LABEL
]
quantities = ['HA', 'UA', 'VA']
# instantiate urs_points of the three mux files.
mux = {}
for quantity, file in zip(quantities, files_in):
mux[quantity] = Read_urs(file)
# Could check that the depth is the same. (hashing)
# handle to a mux file to do depth stuff
a_mux = mux[quantities[0]]
# Convert to utm
lat = a_mux.lonlatdep[:, 1]
long = a_mux.lonlatdep[:, 0]
points_utm, zone = convert_from_latlon_to_utm(latitudes=lat,
longitudes=long)
elevation = a_mux.lonlatdep[:, 2] * -1
# grid (create a mesh from the selected points)
# This mesh has a problem. Triangles are streched over ungridded areas.
# If these areas could be described as holes in pmesh, that would be great.
# I can't just get the user to selection a point in the middle.
# A boundary is needed around these points.
# But if the zone of points is obvious enough auto-segment should do
# a good boundary.
mesh = Mesh()
mesh.add_vertices(points_utm)
mesh.auto_segment(smooth_indents=True, expand_pinch=True)
# To try and avoid alpha shape 'hugging' too much
mesh.auto_segment(mesh.shape.get_alpha() * 1.1)
if hole_points_UTM is not None:
point = ensure_absolute(hole_points_UTM)
mesh.add_hole(point[0], point[1])
try:
mesh.generate_mesh(minimum_triangle_angle=0.0, verbose=False)
except NoTrianglesError:
# This is a bit of a hack, going in and changing the data structure.
mesh.holes = []
mesh.generate_mesh(minimum_triangle_angle=0.0, verbose=False)
mesh_dic = mesh.Mesh2MeshList()
#mesh.export_mesh_file(basename_in + '_168.tsh')
#import sys; sys.exit()
# These are the times of the mux file
mux_times = []
for i in range(a_mux.time_step_count):
mux_times.append(a_mux.time_step * i)
(mux_times_start_i,
mux_times_fin_i) = read_time_from_mux(mux_times, mint, maxt)
times = mux_times[mux_times_start_i:mux_times_fin_i]
if mux_times_start_i == mux_times_fin_i:
# Close the mux files
for quantity, file in zip(quantities, files_in):
mux[quantity].close()
msg = "Due to mint and maxt there's no time info in the boundary SWW."
raise Exception(msg)
# If this raise is removed there is currently no downstream errors
points_utm = ensure_numeric(points_utm)
assert num.alltrue(
ensure_numeric(mesh_dic['generatedpointlist']) == ensure_numeric(
points_utm))
volumes = mesh_dic['generatedtrianglelist']
# Write sww intro and grid stuff.
if basename_out is None:
swwname = basename_in + '.sww'
else:
swwname = basename_out + '.sww'
if verbose: log.critical('Output to %s' % swwname)
outfile = NetCDFFile(swwname, netcdf_mode_w)
# For a different way of doing this, check out tsh2sww
# work out sww_times and the index range this covers
sww = Write_sww(['elevation'], ['stage', 'xmomentum', 'ymomentum'])
sww.store_header(outfile,
times,
len(volumes),
len(points_utm),
verbose=verbose,
sww_precision=netcdf_float)
outfile.mean_stage = mean_stage
outfile.zscale = zscale
sww.store_triangulation(outfile,
points_utm,
volumes,
zone,
new_origin=origin,
verbose=verbose)
sww.store_static_quantities(outfile, elevation=elevation)
if verbose: log.critical('Converting quantities')
# Read in a time slice from each mux file and write it to the SWW file
j = 0
for ha, ua, va in zip(mux['HA'], mux['UA'], mux['VA']):
if j >= mux_times_start_i and j < mux_times_fin_i:
stage = zscale * ha + mean_stage
h = stage - elevation
xmomentum = ua * h
ymomentum = -1 * va * h # -1 since in mux files south is positive.
sww.store_quantities(outfile,
slice_index=j - mux_times_start_i,
verbose=verbose,
stage=stage,
xmomentum=xmomentum,
ymomentum=ymomentum,
sww_precision=num.float)
j += 1
if verbose: sww.verbose_quantities(outfile)
outfile.close()
def sts2sww_mesh(basename_in, basename_out=None,
spatial_thinning=1, verbose=False):
from anuga.mesh_engine.mesh_engine import NoTrianglesError
from anuga.pmesh.mesh import Mesh
if verbose:
print("Starting sts2sww_mesh")
mean_stage=0.
zscale=1.
if (basename_in[:-4]=='.sts'):
stsname = basename_in
else:
stsname = basename_in + '.sts'
if verbose: print("Reading sts NetCDF file: %s" %stsname)
infile = NetCDFFile(stsname, netcdf_mode_r)
cellsize = infile.cellsize
ncols = infile.ncols
nrows = infile.nrows
no_data = infile.no_data
refzone = infile.zone
x_origin = infile.xllcorner
y_origin = infile.yllcorner
origin = num.array([x_origin, y_origin])
x = infile.variables['x'][:]
y = infile.variables['y'][:]
times = infile.variables['time'][:]
wind_speed_full = infile.variables['wind_speed'][:]
wind_angle_full = infile.variables['wind_angle'][:]
pressure_full = infile.variables['barometric_pressure'][:]
infile.close()
number_of_points = nrows*ncols
points_utm = num.zeros((number_of_points,2),num.float)
points_utm[:,0]=x+x_origin
points_utm[:,1]=y+y_origin
thinned_indices=[]
for i in range(number_of_points):
if (old_div(i,ncols)==0 or old_div(i,ncols)==ncols-1 or (old_div(i,ncols))%(spatial_thinning)==0):
if ( i%(spatial_thinning)==0 or i%nrows==0 or i%nrows==nrows-1 ):
thinned_indices.append(i)
#Spatial thinning
points_utm=points_utm[thinned_indices]
number_of_points = points_utm.shape[0]
number_of_timesteps = wind_speed_full.shape[0]
wind_speed = num.empty((number_of_timesteps,number_of_points),dtype=float)
wind_angle = num.empty((number_of_timesteps,number_of_points),dtype=float)
barometric_pressure = num.empty((number_of_timesteps,number_of_points),dtype=float)
if verbose:
print("Total number of points: ", nrows*ncols)
print("Number of thinned points: ", number_of_points)
for i in range(number_of_timesteps):
wind_speed[i] = wind_speed_full[i,thinned_indices]
wind_angle[i] = wind_angle_full[i,thinned_indices]
barometric_pressure[i] = pressure_full[i,thinned_indices]
#P.plot(points_utm[:,0],points_utm[:,1],'ro')
#P.show()
if verbose:
print("Generating sww triangulation of gems data")
mesh = Mesh()
mesh.add_vertices(points_utm)
mesh.auto_segment(smooth_indents=True, expand_pinch=True)
mesh.auto_segment(mesh.shape.get_alpha() * 1.1)
try:
mesh.generate_mesh(minimum_triangle_angle=0.0, verbose=False)
except NoTrianglesError:
# This is a bit of a hack, going in and changing the data structure.
mesh.holes = []
mesh.generate_mesh(minimum_triangle_angle=0.0, verbose=False)
mesh_dic = mesh.Mesh2MeshList()
points_utm=ensure_numeric(points_utm)
assert num.alltrue(ensure_numeric(mesh_dic['generatedpointlist'])
== ensure_numeric(points_utm))
volumes = mesh_dic['generatedtrianglelist']
# Write sww intro and grid stuff.
if (basename_out is not None and basename_out[:-4]=='.sww'):
swwname = basename_out
else:
swwname = basename_in + '.sww'
if verbose: 'Output to %s' % swwname
if verbose:
print("Writing sww wind and pressure field file")
outfile = NetCDFFile(swwname, netcdf_mode_w)
sww = Write_sww([], ['wind_speed','wind_angle','barometric_pressure'])
sww.store_header(outfile, times, len(volumes), len(points_utm),
verbose=verbose, sww_precision='d')
outfile.mean_stage = mean_stage
outfile.zscale = zscale
sww.store_triangulation(outfile, points_utm, volumes,
refzone,
new_origin=origin, #check effect of this line
verbose=verbose)
if verbose:
print('Converting quantities')
# Read in a time slice from the sts file and write it to the SWW file
#print wind_angle[0,:10]
for i in range(len(times)):
sww.store_quantities(outfile,
slice_index=i,
verbose=verbose,
wind_speed=wind_speed[i,:],
wind_angle=wind_angle[i,:],
barometric_pressure=barometric_pressure[i,:],
sww_precision=num.float)
if verbose:
sww.verbose_quantities(outfile)
outfile.close()