def ferret2sww(basename_in, name_out=None, verbose=False, minlat=None, maxlat=None, minlon=None, maxlon=None, mint=None, maxt=None, mean_stage=0, origin=None, zscale=1, fail_on_NaN=True, NaN_filler=0, elevation=None, inverted_bathymetry=True ): #FIXME: Bathymetry should be obtained #from MOST somehow. #Alternatively from elsewhere #or, as a last resort, #specified here. #The value of -100 will work #for the Wollongong tsunami #scenario but is very hacky """Convert MOST and 'Ferret' NetCDF format for wave propagation to sww format native to abstract_2d_finite_volumes. Specify only basename_in and read files of the form basefilename_ha.nc, basefilename_ua.nc, basefilename_va.nc containing relative height, x-velocity and y-velocity, respectively. Also convert latitude and longitude to UTM. All coordinates are assumed to be given in the GDA94 datum. min's and max's: If omitted - full extend is used. To include a value min may equal it, while max must exceed it. Lat and lon are assuemd to be in decimal degrees origin is a 3-tuple with geo referenced UTM coordinates (zone, easting, northing) nc format has values organised as HA[TIME, LATITUDE, LONGITUDE] which means that longitude is the fastest varying dimension (row major order, so to speak) ferret2sww uses grid points as vertices in a triangular grid counting vertices from lower left corner upwards, then right """ from anuga.file.netcdf import NetCDFFile _assert_lat_long(minlat, maxlat, minlon, maxlon) if name_out != None and name_out[-4:] != '.sww': raise IOError('Output file %s should be of type .sww.' % name_out) # Get NetCDF data if verbose: log.critical('Reading files %s_*.nc' % basename_in) # Wave amplitude (cm) file_h = NetCDFFile(basename_in + '_ha.nc', netcdf_mode_r) # Velocity (x) (cm/s) file_u = NetCDFFile(basename_in + '_ua.nc', netcdf_mode_r) # Velocity (y) (cm/s) file_v = NetCDFFile(basename_in + '_va.nc', netcdf_mode_r) # Elevation (z) (m) file_e = NetCDFFile(basename_in + '_e.nc', netcdf_mode_r) if name_out is None: swwname = basename_in + '.sww' else: swwname = name_out # Get dimensions of file_h for dimension in file_h.dimensions.keys(): if dimension[:3] == 'LON': dim_h_longitude = dimension if dimension[:3] == 'LAT': dim_h_latitude = dimension if dimension[:4] == 'TIME': dim_h_time = dimension times = file_h.variables[dim_h_time] latitudes = file_h.variables[dim_h_latitude] longitudes = file_h.variables[dim_h_longitude] kmin, kmax, lmin, lmax = get_min_max_indices(latitudes[:], longitudes[:], minlat, maxlat, minlon, maxlon) # get dimensions for file_e for dimension in file_e.dimensions.keys(): if dimension[:3] == 'LON': dim_e_longitude = dimension if dimension[:3] == 'LAT': dim_e_latitude = dimension # get dimensions for file_u for dimension in file_u.dimensions.keys(): if dimension[:3] == 'LON': dim_u_longitude = dimension if dimension[:3] == 'LAT': dim_u_latitude = dimension # get dimensions for file_v for dimension in file_v.dimensions.keys(): if dimension[:3] == 'LON': dim_v_longitude = dimension if dimension[:3] == 'LAT': dim_v_latitude = dimension # Precision used by most for lat/lon is 4 or 5 decimals e_lat = num.around(file_e.variables[dim_e_latitude][:], 5) e_lon = num.around(file_e.variables[dim_e_longitude][:], 5) # Check that files are compatible assert num.allclose(latitudes, file_u.variables[dim_u_latitude]) assert num.allclose(latitudes, file_v.variables[dim_v_latitude]) assert num.allclose(latitudes, e_lat) assert num.allclose(longitudes, file_u.variables[dim_u_longitude]) assert num.allclose(longitudes, file_v.variables[dim_v_longitude]) assert num.allclose(longitudes, e_lon) if mint is None: jmin = 0 mint = times[0] else: jmin = num.searchsorted(times, mint) # numpy.int32 didn't work in slicing of amplitude below jmin = int(jmin) if maxt is None: jmax = len(times) maxt = times[-1] else: jmax = num.searchsorted(times, maxt) # numpy.int32 didn't work in slicing of amplitude below jmax = int(jmax) kmin, kmax, lmin, lmax = get_min_max_indices(latitudes[:], longitudes[:], minlat, maxlat, minlon, maxlon) times = times[jmin:jmax] latitudes = latitudes[kmin:kmax] longitudes = longitudes[lmin:lmax] if verbose: log.critical('cropping') zname = 'ELEVATION' amplitudes = file_h.variables['HA'][jmin:jmax, kmin:kmax, lmin:lmax] uspeed = file_u.variables['UA'][jmin:jmax, kmin:kmax, lmin:lmax] #Lon vspeed = file_v.variables['VA'][jmin:jmax, kmin:kmax, lmin:lmax] #Lat elevations = file_e.variables[zname][kmin:kmax, lmin:lmax] # Get missing values nan_ha = file_h.variables['HA'].missing_value nan_ua = file_u.variables['UA'].missing_value nan_va = file_v.variables['VA'].missing_value if hasattr(file_e.variables[zname],'missing_value'): nan_e = file_e.variables[zname].missing_value else: nan_e = None # Cleanup missing = (amplitudes == nan_ha) if num.sometrue (missing): if fail_on_NaN: msg = 'NetCDFFile %s contains missing values' \ % basename_in + '_ha.nc' raise DataMissingValuesError, msg else: amplitudes = amplitudes*(missing==0) + missing*NaN_filler missing = (uspeed == nan_ua) if num.sometrue (missing): if fail_on_NaN: msg = 'NetCDFFile %s contains missing values' \ % basename_in + '_ua.nc' raise DataMissingValuesError, msg else: uspeed = uspeed*(missing==0) + missing*NaN_filler missing = (vspeed == nan_va) if num.sometrue (missing): if fail_on_NaN: msg = 'NetCDFFile %s contains missing values' \ % basename_in + '_va.nc' raise DataMissingValuesError, msg else: vspeed = vspeed*(missing==0) + missing*NaN_filler missing = (elevations == nan_e) if num.sometrue (missing): if fail_on_NaN: msg = 'NetCDFFile %s contains missing values' \ % basename_in + '_e.nc' raise DataMissingValuesError, msg else: elevations = elevations*(missing==0) + missing*NaN_filler number_of_times = times.shape[0] number_of_latitudes = latitudes.shape[0] number_of_longitudes = longitudes.shape[0] assert amplitudes.shape[0] == number_of_times assert amplitudes.shape[1] == number_of_latitudes assert amplitudes.shape[2] == number_of_longitudes if verbose: _show_stats((latitudes, longitudes), times, amplitudes, \ (uspeed, vspeed), elevations) # print number_of_latitudes, number_of_longitudes number_of_points = number_of_latitudes * number_of_longitudes number_of_volumes = (number_of_latitudes-1) * (number_of_longitudes-1) * 2 file_h.close() file_u.close() file_v.close() file_e.close() # NetCDF file definition outfile = NetCDFFile(swwname, netcdf_mode_w) description = 'Converted from Ferret files: %s, %s, %s, %s' \ % (basename_in + '_ha.nc', basename_in + '_ua.nc', basename_in + '_va.nc', basename_in + '_e.nc') # Create new file starttime = times[0] sww = Write_sww(['elevation'], ['stage', 'xmomentum', 'ymomentum']) sww.store_header(outfile, times, number_of_volumes, number_of_points, description=description, verbose=verbose, sww_precision=netcdf_float) # Store from anuga.coordinate_transforms.redfearn import redfearn x = num.zeros(number_of_points, num.float) #Easting y = num.zeros(number_of_points, num.float) #Northing if verbose: log.critical('Making triangular grid') # Check zone boundaries refzone, _, _ = redfearn(latitudes[0], longitudes[0]) vertices = {} i = 0 for k, lat in enumerate(latitudes): # Y direction for l, lon in enumerate(longitudes): # X direction vertices[l, k] = i _, easting, northing = redfearn(lat, lon) #msg = 'Zone boundary crossed at longitude =', lon #assert zone == refzone, msg #print '%7.2f %7.2f %8.2f %8.2f' %(lon, lat, easting, northing) x[i] = easting y[i] = northing i += 1 #Construct 2 triangles per 'rectangular' element volumes = [] for l in range(number_of_longitudes-1): # X direction for k in range(number_of_latitudes-1): # Y direction v1 = vertices[l, k+1] v2 = vertices[l, k] v3 = vertices[l+1, k+1] v4 = vertices[l+1, k] volumes.append([v1, v2, v3]) #Upper element volumes.append([v4, v3, v2]) #Lower element volumes = num.array(volumes, num.int) #array default# if origin is None: origin = Geo_reference(refzone, min(x), min(y)) geo_ref = write_NetCDF_georeference(origin, outfile) if elevation is not None: z = elevation else: if inverted_bathymetry: z = -1 * elevations else: z = elevations #FIXME: z should be obtained from MOST and passed in here #FIXME use the Write_sww instance(sww) to write this info z = num.resize(z, outfile.variables['elevation'][:].shape) outfile.variables['x'][:] = x - geo_ref.get_xllcorner() outfile.variables['y'][:] = y - geo_ref.get_yllcorner() #outfile.variables['z'][:] = z #FIXME HACK for bacwards compat. outfile.variables['elevation'][:] = z outfile.variables['volumes'][:] = volumes.astype(num.int32) #For Opteron 64 #Time stepping stage = outfile.variables['stage'] xmomentum = outfile.variables['xmomentum'] ymomentum = outfile.variables['ymomentum'] if verbose: log.critical('Converting quantities') n = len(times) for j in range(n): if verbose and j % ((n+10)/10) == 0: log.critical(' Doing %d of %d' % (j, n)) i = 0 for k in range(number_of_latitudes): # Y direction for l in range(number_of_longitudes): # X direction w = zscale * amplitudes[j, k, l] / 100 + mean_stage stage[j, i] = w h = w - z[i] xmomentum[j, i] = uspeed[j, k, l]/100*h ymomentum[j, i] = vspeed[j, k, l]/100*h i += 1 #outfile.close() #FIXME: Refactor using code from file_function.statistics #Something like print swwstats(swwname) if verbose: time_info = times, starttime, mint, maxt _show_sww_stats(outfile, swwname, geo_ref, time_info) outfile.close()
def urs2sts(basename_in, basename_out=None, weights=None, verbose=False, origin=None, zone=None, central_meridian=None, mean_stage=0.0, zscale=1.0, ordering_filename=None): """Convert URS mux2 format for wave propagation to sts format Also convert latitude and longitude to UTM. All coordinates are assumed to be given in the GDA94 datum origin is a 3-tuple with geo referenced UTM coordinates (zone, easting, northing) inputs: basename_in: list of source file prefixes These are combined with the extensions: WAVEHEIGHT_MUX2_LABEL = '-z-mux2' for stage EAST_VELOCITY_MUX2_LABEL = '-e-mux2' xmomentum NORTH_VELOCITY_MUX2_LABEL = '-n-mux2' and ymomentum to create a 2D list of mux2 file. The rows are associated with each quantity and must have the above extensions the columns are the list of file prefixes. ordering: a .txt file name specifying which mux2 gauge points are to be stored. This is indicated by the index of the gauge in the ordering file. ordering file format: 1st line: 'index,longitude,latitude\n' other lines: index,longitude,latitude If ordering is None or ordering file is empty then all points are taken in the order they appear in the mux2 file. output: basename_out: name of sts file in which mux2 data is stored. NOTE: South is positive in mux files so sign of y-component of velocity is reverted """ import os from anuga.file.netcdf import NetCDFFile from operator import __and__ if not isinstance(basename_in, list): if verbose: log.critical('Reading single source') basename_in = [basename_in] # This is the value used in the mux file format to indicate NAN data # FIXME (Ole): This should be changed everywhere to IEEE NAN when # we upgrade to Numpy NODATA = 99 # Check that basename is a list of strings if not reduce(__and__, map(lambda z: isinstance(z, basestring), basename_in)): msg = 'basename_in must be a string or list of strings' raise Exception, msg # Find the number of sources to be used numSrc = len(basename_in) # A weight must be specified for each source if weights is None: # Default is equal weighting weights = num.ones(numSrc, num.float) / numSrc else: weights = ensure_numeric(weights) msg = 'When combining multiple sources must specify a weight for ' \ 'mux2 source file' assert len(weights) == numSrc, msg if verbose: log.critical('Weights used in urs2sts: %s' % str(weights)) # Check output filename if basename_out is None: msg = 'STS filename must be specified as basename_out ' \ 'in function urs2sts' raise Exception, msg if basename_out.endswith('.sts'): stsname = basename_out else: stsname = basename_out + '.sts' # Create input filenames from basenames and check their existence files_in = [[], [], []] for files in basename_in: files_in[0].append(files + WAVEHEIGHT_MUX2_LABEL), files_in[1].append(files + EAST_VELOCITY_MUX2_LABEL) files_in[2].append(files + NORTH_VELOCITY_MUX2_LABEL) quantities = ['HA', 'UA', 'VA'] # Quantity names used in the MUX2 format for i in range(len(quantities)): for file_in in files_in[i]: if (os.access(file_in, os.R_OK) == 0): msg = 'File %s does not exist or is not accessible' % file_in raise IOError, msg # Establish permutation array if ordering_filename is not None: if verbose is True: log.critical('Reading ordering file %s' % ordering_filename) # Read ordering file try: fid = open(ordering_filename, 'r') file_header = fid.readline().split(',') ordering_lines = fid.readlines() fid.close() except: msg = 'Cannot open %s' % ordering_filename raise Exception, msg reference_header = 'index, longitude, latitude\n' reference_header_split = reference_header.split(',') for i in range(3): if not file_header[i].strip() == reference_header_split[i].strip(): msg = 'File must contain header: ' + reference_header raise Exception, msg if len(ordering_lines) < 2: msg = 'File must contain at least two points' raise Exception, msg permutation = [int(line.split(',')[0]) for line in ordering_lines] permutation = ensure_numeric(permutation) else: permutation = None # Read MUX2 files if (verbose): log.critical('reading mux2 file') mux = {} times_old = 0.0 latitudes_old = 0.0 longitudes_old = 0.0 elevation_old = 0.0 starttime_old = 0.0 for i, quantity in enumerate(quantities): # For each quantity read the associated list of source mux2 file with # extention associated with that quantity times, latitudes, longitudes, elevation, mux[quantity], starttime \ = read_mux2_py(files_in[i], weights, permutation, verbose=verbose) # Check that all quantities have consistent time and space information if quantity != quantities[0]: msg = '%s, %s and %s have inconsistent gauge data' \ % (files_in[0], files_in[1], files_in[2]) assert num.allclose(times, times_old), msg assert num.allclose(latitudes, latitudes_old), msg assert num.allclose(longitudes, longitudes_old), msg assert num.allclose(elevation, elevation_old), msg assert num.allclose(starttime, starttime_old), msg times_old = times latitudes_old = latitudes longitudes_old = longitudes elevation_old = elevation starttime_old = starttime # Self check - can be removed to improve speed #ref_longitudes = [float(line.split(',')[1]) for line in ordering_lines] #ref_latitudes = [float(line.split(',')[2]) for line in ordering_lines] # #msg = 'Longitudes specified in ordering file do not match those ' \ # 'found in mux files. ' \ # 'I got %s instead of %s (only beginning shown)' \ # % (str(longitudes[:10]) + '...', # str(ref_longitudes[:10]) + '...') #assert allclose(longitudes, ref_longitudes), msg # #msg = 'Latitudes specified in ordering file do not match those ' \ # 'found in mux files. ' # 'I got %s instead of %s (only beginning shown)' \ # % (str(latitudes[:10]) + '...', # str(ref_latitudes[:10]) + '...') #assert allclose(latitudes, ref_latitudes), msg # Store timeseries in STS file msg = 'File is empty and or clipped region not in file region' assert len(latitudes > 0), msg number_of_points = latitudes.shape[0] # Number of stations retrieved number_of_times = times.shape[0] # Number of timesteps number_of_latitudes = latitudes.shape[0] # Number latitudes number_of_longitudes = longitudes.shape[0] # Number longitudes # The permutation vector of contains original indices # as given in ordering file or None in which case points # are assigned the trivial indices enumerating them from # 0 to number_of_points-1 if permutation is None: permutation = num.arange(number_of_points, dtype=num.int) # NetCDF file definition outfile = NetCDFFile(stsname, netcdf_mode_w) description = 'Converted from URS mux2 files: %s' % basename_in # Create new file sts = Write_sts() sts.store_header(outfile, times + starttime, number_of_points, description=description, verbose=verbose, sts_precision=netcdf_float) # Store from anuga.coordinate_transforms.redfearn import redfearn x = num.zeros(number_of_points, num.float) # Easting y = num.zeros(number_of_points, num.float) # Northing # Check zone boundaries if zone is None: refzone, _, _ = redfearn(latitudes[0], longitudes[0], central_meridian=central_meridian) else: refzone = zone old_zone = refzone old_easting = 0.0 old_northing = 0.0 for i in range(number_of_points): computed_zone, easting, northing = redfearn( latitudes[i], longitudes[i], zone=zone, central_meridian=central_meridian) x[i] = easting y[i] = northing if computed_zone != refzone: msg = 'All sts gauges need to be in the same zone. \n' msg += 'offending gauge:Zone %d,%.4f, %4f\n' \ % (computed_zone, easting, northing) msg += 'previous gauge:Zone %d,%.4f, %4f' \ % (old_zone, old_easting, old_northing) raise Exception, msg old_zone = computed_zone old_easting = easting old_northing = northing if origin is None: origin = Geo_reference(refzone, min(x), min(y)) geo_ref = write_NetCDF_georeference(origin, outfile) elevation = num.resize(elevation, outfile.variables['elevation'][:].shape) outfile.variables['permutation'][:] = permutation.astype( num.int32) # Opteron 64 outfile.variables['x'][:] = x - geo_ref.get_xllcorner() outfile.variables['y'][:] = y - geo_ref.get_yllcorner() outfile.variables['elevation'][:] = elevation stage = outfile.variables['stage'] xmomentum = outfile.variables['xmomentum'] ymomentum = outfile.variables['ymomentum'] if verbose: log.critical('Converting quantities') for j in range(len(times)): for i in range(number_of_points): ha = mux['HA'][i, j] ua = mux['UA'][i, j] va = mux['VA'][i, j] if ha == NODATA: if verbose: msg = 'Setting nodata value %d to 0 at time = %f, ' \ 'point = %d' % (ha, times[j], i) log.critical(msg) ha = 0.0 ua = 0.0 va = 0.0 w = zscale * ha + mean_stage h = w - elevation[i] stage[j, i] = w xmomentum[j, i] = ua * h ymomentum[j, i] = -va * h # South is positive in mux files outfile.close() if verbose: log.critical('Wrote sts file ' + stsname)
def urs2sts(basename_in, basename_out=None, weights=None, verbose=False, origin=None, zone=None, central_meridian=None, mean_stage=0.0, zscale=1.0, ordering_filename=None): """Convert URS mux2 format for wave propagation to sts format Also convert latitude and longitude to UTM. All coordinates are assumed to be given in the GDA94 datum origin is a 3-tuple with geo referenced UTM coordinates (zone, easting, northing) inputs: basename_in: list of source file prefixes These are combined with the extensions: WAVEHEIGHT_MUX2_LABEL = '-z-mux2' for stage EAST_VELOCITY_MUX2_LABEL = '-e-mux2' xmomentum NORTH_VELOCITY_MUX2_LABEL = '-n-mux2' and ymomentum to create a 2D list of mux2 file. The rows are associated with each quantity and must have the above extensions the columns are the list of file prefixes. ordering: a .txt file name specifying which mux2 gauge points are to be stored. This is indicated by the index of the gauge in the ordering file. ordering file format: 1st line: 'index,longitude,latitude\n' other lines: index,longitude,latitude If ordering is None or ordering file is empty then all points are taken in the order they appear in the mux2 file. output: basename_out: name of sts file in which mux2 data is stored. NOTE: South is positive in mux files so sign of y-component of velocity is reverted """ import os from anuga.file.netcdf import NetCDFFile from operator import __and__ if not isinstance(basename_in, list): if verbose: log.critical('Reading single source') basename_in = [basename_in] # This is the value used in the mux file format to indicate NAN data # FIXME (Ole): This should be changed everywhere to IEEE NAN when # we upgrade to Numpy NODATA = 99 # Check that basename is a list of strings if not reduce(__and__, map(lambda z:isinstance(z,basestring), basename_in)): msg= 'basename_in must be a string or list of strings' raise Exception, msg # Find the number of sources to be used numSrc = len(basename_in) # A weight must be specified for each source if weights is None: # Default is equal weighting weights = num.ones(numSrc, num.float) / numSrc else: weights = ensure_numeric(weights) msg = 'When combining multiple sources must specify a weight for ' \ 'mux2 source file' assert len(weights) == numSrc, msg if verbose: log.critical('Weights used in urs2sts: %s' % str(weights)) # Check output filename if basename_out is None: msg = 'STS filename must be specified as basename_out ' \ 'in function urs2sts' raise Exception, msg if basename_out.endswith('.sts'): stsname = basename_out else: stsname = basename_out + '.sts' # Create input filenames from basenames and check their existence files_in = [[], [], []] for files in basename_in: files_in[0].append(files + WAVEHEIGHT_MUX2_LABEL), files_in[1].append(files + EAST_VELOCITY_MUX2_LABEL) files_in[2].append(files + NORTH_VELOCITY_MUX2_LABEL) quantities = ['HA','UA','VA'] # Quantity names used in the MUX2 format for i in range(len(quantities)): for file_in in files_in[i]: if (os.access(file_in, os.R_OK) == 0): msg = 'File %s does not exist or is not accessible' % file_in raise IOError, msg # Establish permutation array if ordering_filename is not None: if verbose is True: log.critical('Reading ordering file %s' % ordering_filename) # Read ordering file try: fid = open(ordering_filename, 'r') file_header = fid.readline().split(',') ordering_lines = fid.readlines() fid.close() except: msg = 'Cannot open %s' % ordering_filename raise Exception, msg reference_header = 'index, longitude, latitude\n' reference_header_split = reference_header.split(',') for i in range(3): if not file_header[i].strip() == reference_header_split[i].strip(): msg = 'File must contain header: ' + reference_header raise Exception, msg if len(ordering_lines) < 2: msg = 'File must contain at least two points' raise Exception, msg permutation = [int(line.split(',')[0]) for line in ordering_lines] permutation = ensure_numeric(permutation) else: permutation = None # Read MUX2 files if (verbose): log.critical('reading mux2 file') mux={} times_old = 0.0 latitudes_old = 0.0 longitudes_old = 0.0 elevation_old = 0.0 starttime_old = 0.0 for i, quantity in enumerate(quantities): # For each quantity read the associated list of source mux2 file with # extention associated with that quantity times, latitudes, longitudes, elevation, mux[quantity], starttime \ = read_mux2_py(files_in[i], weights, permutation, verbose=verbose) # Check that all quantities have consistent time and space information if quantity != quantities[0]: msg = '%s, %s and %s have inconsistent gauge data' \ % (files_in[0], files_in[1], files_in[2]) assert num.allclose(times, times_old), msg assert num.allclose(latitudes, latitudes_old), msg assert num.allclose(longitudes, longitudes_old), msg assert num.allclose(elevation, elevation_old), msg assert num.allclose(starttime, starttime_old), msg times_old = times latitudes_old = latitudes longitudes_old = longitudes elevation_old = elevation starttime_old = starttime # Self check - can be removed to improve speed #ref_longitudes = [float(line.split(',')[1]) for line in ordering_lines] #ref_latitudes = [float(line.split(',')[2]) for line in ordering_lines] # #msg = 'Longitudes specified in ordering file do not match those ' \ # 'found in mux files. ' \ # 'I got %s instead of %s (only beginning shown)' \ # % (str(longitudes[:10]) + '...', # str(ref_longitudes[:10]) + '...') #assert allclose(longitudes, ref_longitudes), msg # #msg = 'Latitudes specified in ordering file do not match those ' \ # 'found in mux files. ' # 'I got %s instead of %s (only beginning shown)' \ # % (str(latitudes[:10]) + '...', # str(ref_latitudes[:10]) + '...') #assert allclose(latitudes, ref_latitudes), msg # Store timeseries in STS file msg = 'File is empty and or clipped region not in file region' assert len(latitudes > 0), msg number_of_points = latitudes.shape[0] # Number of stations retrieved number_of_times = times.shape[0] # Number of timesteps number_of_latitudes = latitudes.shape[0] # Number latitudes number_of_longitudes = longitudes.shape[0] # Number longitudes # The permutation vector of contains original indices # as given in ordering file or None in which case points # are assigned the trivial indices enumerating them from # 0 to number_of_points-1 if permutation is None: permutation = num.arange(number_of_points, dtype=num.int) # NetCDF file definition outfile = NetCDFFile(stsname, netcdf_mode_w) description = 'Converted from URS mux2 files: %s' % basename_in # Create new file sts = Write_sts() sts.store_header(outfile, times+starttime, number_of_points, description=description, verbose=verbose, sts_precision=netcdf_float) # Store from anuga.coordinate_transforms.redfearn import redfearn x = num.zeros(number_of_points, num.float) # Easting y = num.zeros(number_of_points, num.float) # Northing # Check zone boundaries if zone is None: refzone, _, _ = redfearn(latitudes[0], longitudes[0], central_meridian=central_meridian) else: refzone = zone old_zone = refzone old_easting = 0.0 old_northing = 0.0 for i in range(number_of_points): computed_zone, easting, northing = redfearn(latitudes[i], longitudes[i], zone=zone, central_meridian=central_meridian) x[i] = easting y[i] = northing if computed_zone != refzone: msg = 'All sts gauges need to be in the same zone. \n' msg += 'offending gauge:Zone %d,%.4f, %4f\n' \ % (computed_zone, easting, northing) msg += 'previous gauge:Zone %d,%.4f, %4f' \ % (old_zone, old_easting, old_northing) raise Exception, msg old_zone = computed_zone old_easting = easting old_northing = northing if origin is None: origin = Geo_reference(refzone, min(x), min(y)) geo_ref = write_NetCDF_georeference(origin, outfile) elevation = num.resize(elevation, outfile.variables['elevation'][:].shape) outfile.variables['permutation'][:] = permutation.astype(num.int32) # Opteron 64 outfile.variables['x'][:] = x - geo_ref.get_xllcorner() outfile.variables['y'][:] = y - geo_ref.get_yllcorner() outfile.variables['elevation'][:] = elevation stage = outfile.variables['stage'] xmomentum = outfile.variables['xmomentum'] ymomentum = outfile.variables['ymomentum'] if verbose: log.critical('Converting quantities') for j in range(len(times)): for i in range(number_of_points): ha = mux['HA'][i,j] ua = mux['UA'][i,j] va = mux['VA'][i,j] if ha == NODATA: if verbose: msg = 'Setting nodata value %d to 0 at time = %f, ' \ 'point = %d' % (ha, times[j], i) log.critical(msg) ha = 0.0 ua = 0.0 va = 0.0 w = zscale*ha + mean_stage h = w - elevation[i] stage[j,i] = w xmomentum[j,i] = ua * h ymomentum[j,i] = -va * h # South is positive in mux files outfile.close() if verbose: log.critical('Wrote sts file ' + stsname)
def ferret2sww( basename_in, name_out=None, verbose=False, minlat=None, maxlat=None, minlon=None, maxlon=None, mint=None, maxt=None, mean_stage=0, origin=None, zscale=1, fail_on_NaN=True, NaN_filler=0, elevation=None, inverted_bathymetry=True): #FIXME: Bathymetry should be obtained #from MOST somehow. #Alternatively from elsewhere #or, as a last resort, #specified here. #The value of -100 will work #for the Wollongong tsunami #scenario but is very hacky """Convert MOST and 'Ferret' NetCDF format for wave propagation to sww format native to abstract_2d_finite_volumes. Specify only basename_in and read files of the form basefilename_ha.nc, basefilename_ua.nc, basefilename_va.nc containing relative height, x-velocity and y-velocity, respectively. Also convert latitude and longitude to UTM. All coordinates are assumed to be given in the GDA94 datum. min's and max's: If omitted - full extend is used. To include a value min may equal it, while max must exceed it. Lat and lon are assuemd to be in decimal degrees origin is a 3-tuple with geo referenced UTM coordinates (zone, easting, northing) nc format has values organised as HA[TIME, LATITUDE, LONGITUDE] which means that longitude is the fastest varying dimension (row major order, so to speak) ferret2sww uses grid points as vertices in a triangular grid counting vertices from lower left corner upwards, then right """ from anuga.file.netcdf import NetCDFFile _assert_lat_long(minlat, maxlat, minlon, maxlon) if name_out != None and name_out[-4:] != '.sww': raise IOError('Output file %s should be of type .sww.' % name_out) # Get NetCDF data if verbose: log.critical('Reading files %s_*.nc' % basename_in) # Wave amplitude (cm) file_h = NetCDFFile(basename_in + '_ha.nc', netcdf_mode_r) # Velocity (x) (cm/s) file_u = NetCDFFile(basename_in + '_ua.nc', netcdf_mode_r) # Velocity (y) (cm/s) file_v = NetCDFFile(basename_in + '_va.nc', netcdf_mode_r) # Elevation (z) (m) file_e = NetCDFFile(basename_in + '_e.nc', netcdf_mode_r) if name_out is None: swwname = basename_in + '.sww' else: swwname = name_out # Get dimensions of file_h for dimension in list(file_h.dimensions.keys()): if dimension[:3] == 'LON': dim_h_longitude = dimension if dimension[:3] == 'LAT': dim_h_latitude = dimension if dimension[:4] == 'TIME': dim_h_time = dimension times = file_h.variables[dim_h_time] latitudes = file_h.variables[dim_h_latitude] longitudes = file_h.variables[dim_h_longitude] kmin, kmax, lmin, lmax = get_min_max_indices(latitudes[:], longitudes[:], minlat, maxlat, minlon, maxlon) # get dimensions for file_e for dimension in list(file_e.dimensions.keys()): if dimension[:3] == 'LON': dim_e_longitude = dimension if dimension[:3] == 'LAT': dim_e_latitude = dimension # get dimensions for file_u for dimension in list(file_u.dimensions.keys()): if dimension[:3] == 'LON': dim_u_longitude = dimension if dimension[:3] == 'LAT': dim_u_latitude = dimension # get dimensions for file_v for dimension in list(file_v.dimensions.keys()): if dimension[:3] == 'LON': dim_v_longitude = dimension if dimension[:3] == 'LAT': dim_v_latitude = dimension # Precision used by most for lat/lon is 4 or 5 decimals e_lat = num.around(file_e.variables[dim_e_latitude][:], 5) e_lon = num.around(file_e.variables[dim_e_longitude][:], 5) # Check that files are compatible assert num.allclose(latitudes, file_u.variables[dim_u_latitude]) assert num.allclose(latitudes, file_v.variables[dim_v_latitude]) assert num.allclose(latitudes, e_lat) assert num.allclose(longitudes, file_u.variables[dim_u_longitude]) assert num.allclose(longitudes, file_v.variables[dim_v_longitude]) assert num.allclose(longitudes, e_lon) if mint is None: jmin = 0 mint = times[0] else: jmin = num.searchsorted(times, mint) # numpy.int32 didn't work in slicing of amplitude below jmin = int(jmin) if maxt is None: jmax = len(times) maxt = times[-1] else: jmax = num.searchsorted(times, maxt) # numpy.int32 didn't work in slicing of amplitude below jmax = int(jmax) kmin, kmax, lmin, lmax = get_min_max_indices(latitudes[:], longitudes[:], minlat, maxlat, minlon, maxlon) times = times[jmin:jmax] latitudes = latitudes[kmin:kmax] longitudes = longitudes[lmin:lmax] if verbose: log.critical('cropping') zname = 'ELEVATION' amplitudes = file_h.variables['HA'][jmin:jmax, kmin:kmax, lmin:lmax] uspeed = file_u.variables['UA'][jmin:jmax, kmin:kmax, lmin:lmax] #Lon vspeed = file_v.variables['VA'][jmin:jmax, kmin:kmax, lmin:lmax] #Lat elevations = file_e.variables[zname][kmin:kmax, lmin:lmax] # Get missing values nan_ha = file_h.variables['HA'].missing_value nan_ua = file_u.variables['UA'].missing_value nan_va = file_v.variables['VA'].missing_value if hasattr(file_e.variables[zname], 'missing_value'): nan_e = file_e.variables[zname].missing_value else: nan_e = None # Cleanup missing = (amplitudes == nan_ha) if num.sometrue(missing): if fail_on_NaN: msg = 'NetCDFFile %s contains missing values' \ % basename_in + '_ha.nc' raise_(DataMissingValuesError, msg) else: amplitudes = amplitudes * (missing == 0) + missing * NaN_filler missing = (uspeed == nan_ua) if num.sometrue(missing): if fail_on_NaN: msg = 'NetCDFFile %s contains missing values' \ % basename_in + '_ua.nc' raise_(DataMissingValuesError, msg) else: uspeed = uspeed * (missing == 0) + missing * NaN_filler missing = (vspeed == nan_va) if num.sometrue(missing): if fail_on_NaN: msg = 'NetCDFFile %s contains missing values' \ % basename_in + '_va.nc' raise_(DataMissingValuesError, msg) else: vspeed = vspeed * (missing == 0) + missing * NaN_filler missing = (elevations == nan_e) if num.sometrue(missing): if fail_on_NaN: msg = 'NetCDFFile %s contains missing values' \ % basename_in + '_e.nc' raise_(DataMissingValuesError, msg) else: elevations = elevations * (missing == 0) + missing * NaN_filler number_of_times = times.shape[0] number_of_latitudes = latitudes.shape[0] number_of_longitudes = longitudes.shape[0] assert amplitudes.shape[0] == number_of_times assert amplitudes.shape[1] == number_of_latitudes assert amplitudes.shape[2] == number_of_longitudes if verbose: _show_stats((latitudes, longitudes), times, amplitudes, \ (uspeed, vspeed), elevations) # print number_of_latitudes, number_of_longitudes number_of_points = number_of_latitudes * number_of_longitudes number_of_volumes = (number_of_latitudes - 1) * (number_of_longitudes - 1) * 2 file_h.close() file_u.close() file_v.close() file_e.close() # NetCDF file definition outfile = NetCDFFile(swwname, netcdf_mode_w) description = 'Converted from Ferret files: %s, %s, %s, %s' \ % (basename_in + '_ha.nc', basename_in + '_ua.nc', basename_in + '_va.nc', basename_in + '_e.nc') # Create new file starttime = times[0] sww = Write_sww(['elevation'], ['stage', 'xmomentum', 'ymomentum']) sww.store_header(outfile, times, number_of_volumes, number_of_points, description=description, verbose=verbose, sww_precision=netcdf_float) # Store from anuga.coordinate_transforms.redfearn import redfearn x = num.zeros(number_of_points, num.float) #Easting y = num.zeros(number_of_points, num.float) #Northing if verbose: log.critical('Making triangular grid') # Check zone boundaries refzone, _, _ = redfearn(latitudes[0], longitudes[0]) vertices = {} i = 0 for k, lat in enumerate(latitudes): # Y direction for l, lon in enumerate(longitudes): # X direction vertices[l, k] = i _, easting, northing = redfearn(lat, lon) #msg = 'Zone boundary crossed at longitude =', lon #assert zone == refzone, msg #print '%7.2f %7.2f %8.2f %8.2f' %(lon, lat, easting, northing) x[i] = easting y[i] = northing i += 1 #Construct 2 triangles per 'rectangular' element volumes = [] for l in range(number_of_longitudes - 1): # X direction for k in range(number_of_latitudes - 1): # Y direction v1 = vertices[l, k + 1] v2 = vertices[l, k] v3 = vertices[l + 1, k + 1] v4 = vertices[l + 1, k] volumes.append([v1, v2, v3]) #Upper element volumes.append([v4, v3, v2]) #Lower element volumes = num.array(volumes, num.int) #array default# if origin is None: origin = Geo_reference(refzone, min(x), min(y)) geo_ref = write_NetCDF_georeference(origin, outfile) if elevation is not None: z = elevation else: if inverted_bathymetry: z = -1 * elevations else: z = elevations #FIXME: z should be obtained from MOST and passed in here #FIXME use the Write_sww instance(sww) to write this info z = num.resize(z, outfile.variables['elevation'][:].shape) outfile.variables['x'][:] = x - geo_ref.get_xllcorner() outfile.variables['y'][:] = y - geo_ref.get_yllcorner() #outfile.variables['z'][:] = z #FIXME HACK for bacwards compat. outfile.variables['elevation'][:] = z outfile.variables['volumes'][:] = volumes.astype( num.int32) #For Opteron 64 #Time stepping stage = outfile.variables['stage'] xmomentum = outfile.variables['xmomentum'] ymomentum = outfile.variables['ymomentum'] if verbose: log.critical('Converting quantities') n = len(times) for j in range(n): if verbose and j % (old_div((n + 10), 10)) == 0: log.critical(' Doing %d of %d' % (j, n)) i = 0 for k in range(number_of_latitudes): # Y direction for l in range(number_of_longitudes): # X direction w = old_div(zscale * amplitudes[j, k, l], 100) + mean_stage stage[j, i] = w h = w - z[i] xmomentum[j, i] = old_div(uspeed[j, k, l], 100) * h ymomentum[j, i] = old_div(vspeed[j, k, l], 100) * h i += 1 #outfile.close() #FIXME: Refactor using code from file_function.statistics #Something like print swwstats(swwname) if verbose: time_info = times, starttime, mint, maxt _show_sww_stats(outfile, swwname, geo_ref, time_info) outfile.close()