def prepare_timeboundary(filename, verbose = False): """Convert benchmark 2 time series to NetCDF tms file. This is a 'throw-away' code taylor made for files like 'Benchmark_2_input.txt' from the LWRU2 benchmark """ from anuga.file.netcdf import NetCDFFile if verbose: print 'Creating', filename # Read the ascii (.txt) version of this file fid = open(filename[:-4] + '.txt') # Skip first line line = fid.readline() # Read remaining lines lines = fid.readlines() fid.close() N = len(lines) T = num.zeros(N, num.float) #Time Q = num.zeros(N, num.float) #Values for i, line in enumerate(lines): fields = line.split() T[i] = float(fields[0]) Q[i] = float(fields[1]) # Create tms NetCDF file fid = NetCDFFile(filename, 'w') fid.institution = 'Geoscience Australia' fid.description = 'Input wave for Benchmark 2' fid.starttime = 0.0 fid.createDimension('number_of_timesteps', len(T)) fid.createVariable('time', netcdf_float, ('number_of_timesteps',)) fid.variables['time'][:] = T fid.createVariable('stage', netcdf_float, ('number_of_timesteps',)) fid.variables['stage'][:] = Q[:] fid.createVariable('xmomentum', netcdf_float, ('number_of_timesteps',)) fid.variables['xmomentum'][:] = 0.0 fid.createVariable('ymomentum', netcdf_float, ('number_of_timesteps',)) fid.variables['ymomentum'][:] = 0.0 fid.close()
def prepare_timeboundary(filename, verbose=False): """Convert benchmark 2 time series to NetCDF tms file. This is a 'throw-away' code taylor made for files like 'Benchmark_2_input.txt' from the LWRU2 benchmark """ from anuga.file.netcdf import NetCDFFile if verbose: print 'Creating', filename # Read the ascii (.txt) version of this file fid = open(filename[:-4] + '.txt') # Skip first line line = fid.readline() # Read remaining lines lines = fid.readlines() fid.close() N = len(lines) T = num.zeros(N, num.float) #Time Q = num.zeros(N, num.float) #Values for i, line in enumerate(lines): fields = line.split() T[i] = float(fields[0]) Q[i] = float(fields[1]) # Create tms NetCDF file fid = NetCDFFile(filename, 'w') fid.institution = 'Geoscience Australia' fid.description = 'Input wave for Benchmark 2' fid.starttime = 0.0 fid.createDimension('number_of_timesteps', len(T)) fid.createVariable('time', netcdf_float, ('number_of_timesteps', )) fid.variables['time'][:] = T fid.createVariable('stage', netcdf_float, ('number_of_timesteps', )) fid.variables['stage'][:] = Q[:] fid.createVariable('xmomentum', netcdf_float, ('number_of_timesteps', )) fid.variables['xmomentum'][:] = 0.0 fid.createVariable('ymomentum', netcdf_float, ('number_of_timesteps', )) fid.variables['ymomentum'][:] = 0.0 fid.close()
def helper_write_msh_file(self, filename, l): # open the NetCDF file fd = NetCDFFile(filename, netcdf_mode_w) fd.description = 'Test file - string arrays' # convert list of strings to num.array al = num.array(string_to_char(l), num.character) # write the list fd.createDimension('num_of_strings', al.shape[0]) fd.createDimension('size_of_strings', al.shape[1]) var = fd.createVariable('strings', netcdf_char, ('num_of_strings', 'size_of_strings')) var[:] = al fd.close()
def esri2sww( bath_dir, elevation_dir, ucur_dir, vcur_dir, sww_file, minlat=None, maxlat=None, minlon=None, maxlon=None, zscale=1, mean_stage=0, fail_on_NaN=True, elevation_NaN_filler=0, bath_prefix="ba", elevation_prefix="el", verbose=False, ): """ Produce an sww boundary file, from esri ascii data from CSIRO. Also convert latitude and longitude to UTM. All coordinates are assumed to be given in the GDA94 datum. assume: All files are in esri ascii format 4 types of information bathymetry elevation u velocity v velocity Assumptions The metadata of all the files is the same Each type is in a seperate directory One bath file with extention .000 The time period is less than 24hrs and uniform. """ from anuga.file.netcdf import NetCDFFile from anuga.coordinate_transforms.redfearn import redfearn if sww_file[-4:] != ".sww": raise IOError("Output file %s should be of type .sww." % sww_file) # So if we want to change the precision it's done here precision = netcdf_float # go in to the bath dir and load the only file, bath_files = os.listdir(bath_dir) bath_file = bath_files[0] bath_dir_file = bath_dir + os.sep + bath_file bath_metadata, bath_grid = _read_asc(bath_dir_file) # Use the date.time of the bath file as a basis for # the start time for other files base_start = bath_file[-12:] # go into the elevation dir and load the 000 file elevation_dir_file = elevation_dir + os.sep + elevation_prefix + base_start elevation_files = os.listdir(elevation_dir) ucur_files = os.listdir(ucur_dir) vcur_files = os.listdir(vcur_dir) elevation_files.sort() # the first elevation file should be the # file with the same base name as the bath data assert elevation_files[0] == "el" + base_start number_of_latitudes = bath_grid.shape[0] number_of_longitudes = bath_grid.shape[1] number_of_volumes = (number_of_latitudes - 1) * (number_of_longitudes - 1) * 2 longitudes = [bath_metadata["xllcorner"] + x * bath_metadata["cellsize"] for x in range(number_of_longitudes)] latitudes = [bath_metadata["yllcorner"] + y * bath_metadata["cellsize"] for y in range(number_of_latitudes)] # reverse order of lat, so the first lat represents the first grid row latitudes.reverse() kmin, kmax, lmin, lmax = get_min_max_indices( latitudes[:], longitudes[:], minlat=minlat, maxlat=maxlat, minlon=minlon, maxlon=maxlon ) bath_grid = bath_grid[kmin:kmax, lmin:lmax] latitudes = latitudes[kmin:kmax] longitudes = longitudes[lmin:lmax] number_of_latitudes = len(latitudes) number_of_longitudes = len(longitudes) number_of_times = len(os.listdir(elevation_dir)) number_of_points = number_of_latitudes * number_of_longitudes number_of_volumes = (number_of_latitudes - 1) * (number_of_longitudes - 1) * 2 # Work out the times if len(elevation_files) > 1: # Assume: The time period is less than 24hrs. time_period = (int(elevation_files[1][-3:]) - int(elevation_files[0][-3:])) * 60 * 60 times = [x * time_period for x in range(len(elevation_files))] else: times = [0.0] if verbose: log.critical("------------------------------------------------") log.critical("Statistics:") log.critical(" Extent (lat/lon):") log.critical(" lat in [%f, %f], len(lat) == %d" % (min(latitudes), max(latitudes), len(latitudes))) log.critical(" lon in [%f, %f], len(lon) == %d" % (min(longitudes), max(longitudes), len(longitudes))) log.critical(" t in [%f, %f], len(t) == %d" % (min(times), max(times), len(times))) ######### WRITE THE SWW FILE ############# # NetCDF file definition outfile = NetCDFFile(sww_file, netcdf_mode_w) # Create new file outfile.institution = "Geoscience Australia" outfile.description = "Converted from XXX" # For sww compatibility outfile.smoothing = "Yes" outfile.order = 1 # Start time in seconds since the epoch (midnight 1/1/1970) outfile.starttime = starttime = times[0] # dimension definitions outfile.createDimension("number_of_volumes", number_of_volumes) outfile.createDimension("number_of_vertices", 3) outfile.createDimension("number_of_points", number_of_points) outfile.createDimension("number_of_timesteps", number_of_times) # variable definitions outfile.createVariable("x", precision, ("number_of_points",)) outfile.createVariable("y", precision, ("number_of_points",)) outfile.createVariable("elevation", precision, ("number_of_points",)) # FIXME: Backwards compatibility # outfile.createVariable('z', precision, ('number_of_points',)) ################################# outfile.createVariable("volumes", netcdf_int, ("number_of_volumes", "number_of_vertices")) outfile.createVariable("time", precision, ("number_of_timesteps",)) outfile.createVariable("stage", precision, ("number_of_timesteps", "number_of_points")) outfile.createVariable("xmomentum", precision, ("number_of_timesteps", "number_of_points")) outfile.createVariable("ymomentum", precision, ("number_of_timesteps", "number_of_points")) # Store from anuga.coordinate_transforms.redfearn import redfearn x = num.zeros(number_of_points, num.float) # Easting y = num.zeros(number_of_points, num.float) # Northing if verbose: log.critical("Making triangular grid") # Get zone of 1st point. refzone, _, _ = redfearn(latitudes[0], longitudes[0]) vertices = {} i = 0 for k, lat in enumerate(latitudes): for l, lon in enumerate(longitudes): vertices[l, k] = i zone, easting, northing = redfearn(lat, lon) # msg = 'Zone boundary crossed at longitude =', lon # assert zone == refzone, msg # print '%7.2f %7.2f %8.2f %8.2f' %(lon, lat, easting, northing) x[i] = easting y[i] = northing i += 1 # Construct 2 triangles per 'rectangular' element volumes = [] for l in range(number_of_longitudes - 1): # X direction for k in range(number_of_latitudes - 1): # Y direction v1 = vertices[l, k + 1] v2 = vertices[l, k] v3 = vertices[l + 1, k + 1] v4 = vertices[l + 1, k] # Note, this is different to the ferrit2sww code # since the order of the lats is reversed. volumes.append([v1, v3, v2]) # Upper element volumes.append([v4, v2, v3]) # Lower element volumes = num.array(volumes, num.int) # array default# geo_ref = Geo_reference(refzone, min(x), min(y)) geo_ref.write_NetCDF(outfile) # This will put the geo ref in the middle # geo_ref = Geo_reference(refzone, (max(x)+min(x))/2., (max(x)+min(y))/2.) if verbose: log.critical("------------------------------------------------") log.critical("More Statistics:") log.critical(" Extent (/lon):") log.critical(" x in [%f, %f], len(lat) == %d" % (min(x), max(x), len(x))) log.critical(" y in [%f, %f], len(lon) == %d" % (min(y), max(y), len(y))) log.critical("geo_ref: ", geo_ref) z = num.resize(bath_grid, outfile.variables["elevation"][:].shape) outfile.variables["x"][:] = x - geo_ref.get_xllcorner() outfile.variables["y"][:] = y - geo_ref.get_yllcorner() # FIXME (Ole): Remove once viewer has been recompiled and changed # to use elevation instead of z # outfile.variables['z'][:] = z outfile.variables["elevation"][:] = z outfile.variables["volumes"][:] = volumes.astype(num.int32) # On Opteron 64 stage = outfile.variables["stage"] xmomentum = outfile.variables["xmomentum"] ymomentum = outfile.variables["ymomentum"] outfile.variables["time"][:] = times # Store time relative if verbose: log.critical("Converting quantities") n = number_of_times for j in range(number_of_times): # load in files elevation_meta, elevation_grid = _read_asc(elevation_dir + os.sep + elevation_files[j]) _, u_momentum_grid = _read_asc(ucur_dir + os.sep + ucur_files[j]) _, v_momentum_grid = _read_asc(vcur_dir + os.sep + vcur_files[j]) # cut matrix to desired size elevation_grid = elevation_grid[kmin:kmax, lmin:lmax] u_momentum_grid = u_momentum_grid[kmin:kmax, lmin:lmax] v_momentum_grid = v_momentum_grid[kmin:kmax, lmin:lmax] # handle missing values missing = elevation_grid == elevation_meta["NODATA_value"] if num.sometrue(missing): if fail_on_NaN: msg = "File %s contains missing values" % (elevation_files[j]) raise DataMissingValuesError, msg else: elevation_grid = elevation_grid * (missing == 0) + missing * elevation_NaN_filler if verbose and j % ((n + 10) / 10) == 0: log.critical(" Doing %d of %d" % (j, n)) i = 0 for k in range(number_of_latitudes): # Y direction for l in range(number_of_longitudes): # X direction w = zscale * elevation_grid[k, l] + mean_stage stage[j, i] = w h = w - z[i] xmomentum[j, i] = u_momentum_grid[k, l] * h ymomentum[j, i] = v_momentum_grid[k, l] * h i += 1 outfile.close()
def test_decimate_dem(self): """Test decimation of dem file """ import os from anuga.file.netcdf import NetCDFFile #Write test dem file root = 'decdemtest' filename = root + '.dem' fid = NetCDFFile(filename, netcdf_mode_w) fid.institution = 'Geoscience Australia' fid.description = 'NetCDF DEM format for compact and portable ' +\ 'storage of spatial point data' nrows = 15 ncols = 18 fid.ncols = ncols fid.nrows = nrows fid.xllcorner = 2000.5 fid.yllcorner = 3000.5 fid.cellsize = 25 fid.NODATA_value = -9999 fid.zone = 56 fid.false_easting = 0.0 fid.false_northing = 0.0 fid.projection = 'UTM' fid.datum = 'WGS84' fid.units = 'METERS' fid.createDimension('number_of_points', nrows * ncols) fid.createVariable('elevation', netcdf_float, ('number_of_points', )) elevation = fid.variables['elevation'] elevation[:] = (num.arange(nrows * ncols)) fid.close() #generate the elevation values expected in the decimated file ref_elevation = [ (0 + 1 + 2 + 18 + 19 + 20 + 36 + 37 + 38) / 9.0, (4 + 5 + 6 + 22 + 23 + 24 + 40 + 41 + 42) / 9.0, (8 + 9 + 10 + 26 + 27 + 28 + 44 + 45 + 46) / 9.0, (12 + 13 + 14 + 30 + 31 + 32 + 48 + 49 + 50) / 9.0, (72 + 73 + 74 + 90 + 91 + 92 + 108 + 109 + 110) / 9.0, (76 + 77 + 78 + 94 + 95 + 96 + 112 + 113 + 114) / 9.0, (80 + 81 + 82 + 98 + 99 + 100 + 116 + 117 + 118) / 9.0, (84 + 85 + 86 + 102 + 103 + 104 + 120 + 121 + 122) / 9.0, (144 + 145 + 146 + 162 + 163 + 164 + 180 + 181 + 182) / 9.0, (148 + 149 + 150 + 166 + 167 + 168 + 184 + 185 + 186) / 9.0, (152 + 153 + 154 + 170 + 171 + 172 + 188 + 189 + 190) / 9.0, (156 + 157 + 158 + 174 + 175 + 176 + 192 + 193 + 194) / 9.0, (216 + 217 + 218 + 234 + 235 + 236 + 252 + 253 + 254) / 9.0, (220 + 221 + 222 + 238 + 239 + 240 + 256 + 257 + 258) / 9.0, (224 + 225 + 226 + 242 + 243 + 244 + 260 + 261 + 262) / 9.0, (228 + 229 + 230 + 246 + 247 + 248 + 264 + 265 + 266) / 9.0 ] # generate a stencil for computing the decimated values stencil = num.ones((3, 3), num.float) / 9.0 dem2dem(filename, stencil=stencil, cellsize_new=100) # Open decimated NetCDF file fid = NetCDFFile(root + '_100.dem', netcdf_mode_r) # Get decimated elevation elevation = fid.variables['elevation'] # Check values assert num.allclose(elevation, ref_elevation) # Cleanup fid.close() os.remove(root + '.dem') os.remove(root + '_100.dem')
def _generic_convert_dem_from_ascii2netcdf(name_in, name_out=None, quantity_name=None, verbose=False): """Read raster from the following ASCII format (.asc) Internal function. See public function convert_dem_from_ascii2netcdf for details. """ import os from anuga.file.netcdf import NetCDFFile root = name_in[:-4] # Read Meta data if verbose: log.critical('Reading METADATA from %s' % (root + '.prj')) metadatafile = open(root + '.prj') metalines = metadatafile.readlines() metadatafile.close() L = metalines[0].strip().split() assert L[0].strip().lower() == 'projection' projection = L[1].strip() #TEXT L = metalines[1].strip().split() assert L[0].strip().lower() == 'zone' zone = int(L[1].strip()) L = metalines[2].strip().split() assert L[0].strip().lower() == 'datum' datum = L[1].strip() #TEXT L = metalines[3].strip().split() assert L[0].strip().lower() == 'zunits' #IGNORE zunits = L[1].strip() #TEXT L = metalines[4].strip().split() assert L[0].strip().lower() == 'units' units = L[1].strip() #TEXT L = metalines[5].strip().split() assert L[0].strip().lower() == 'spheroid' #IGNORE spheroid = L[1].strip() #TEXT L = metalines[6].strip().split() assert L[0].strip().lower() == 'xshift' false_easting = float(L[1].strip()) L = metalines[7].strip().split() assert L[0].strip().lower() == 'yshift' false_northing = float(L[1].strip()) if name_in[-4:] != '.asc': raise IOError('Input file %s should be of type .asc.' % name_in) #Read DEM data datafile = open(name_in) if verbose: log.critical('Reading raster from %s' % (name_in)) lines = datafile.readlines() datafile.close() if verbose: log.critical('Got %d lines' % len(lines)) ncols = int(lines[0].split()[1].strip()) nrows = int(lines[1].split()[1].strip()) # Do cellsize (line 4) before line 2 and 3 cellsize = float(lines[4].split()[1].strip()) # Checks suggested by Joaquim Luis # Our internal representation of xllcorner # and yllcorner is non-standard. xref = lines[2].split() if xref[0].strip() == 'xllcorner': xllcorner = float(xref[1].strip()) # + 0.5*cellsize # Correct offset elif xref[0].strip() == 'xllcenter': xllcorner = float(xref[1].strip()) else: msg = 'Unknown keyword: %s' % xref[0].strip() raise Exception, msg yref = lines[3].split() if yref[0].strip() == 'yllcorner': yllcorner = float(yref[1].strip()) # + 0.5*cellsize # Correct offset elif yref[0].strip() == 'yllcenter': yllcorner = float(yref[1].strip()) else: msg = 'Unknown keyword: %s' % yref[0].strip() raise Exception, msg NODATA_value = int(float(lines[5].split()[1].strip())) assert len(lines) == nrows + 6 if name_out == None: netcdfname = name_in[:-4] + '.dem' else: netcdfname = name_out + '.dem' if verbose: log.critical('Store to NetCDF file %s' % netcdfname) # NetCDF file definition fid = NetCDFFile(netcdfname, netcdf_mode_w) #Create new file fid.institution = 'Geoscience Australia' fid.description = 'NetCDF DEM format for compact and portable storage ' \ 'of spatial point data' fid.ncols = ncols fid.nrows = nrows fid.xllcorner = xllcorner fid.yllcorner = yllcorner fid.cellsize = cellsize fid.NODATA_value = NODATA_value fid.zone = zone fid.false_easting = false_easting fid.false_northing = false_northing fid.projection = projection fid.datum = datum fid.units = units # dimension definitions fid.createDimension('number_of_rows', nrows) fid.createDimension('number_of_columns', ncols) # variable definitions fid.createVariable(quantity_name, netcdf_float, ('number_of_rows', 'number_of_columns')) # Get handles to the variables elevation = fid.variables[quantity_name] #Store data import numpy datafile = open(name_in) elevation[:, :] = numpy.loadtxt(datafile, skiprows=6) datafile.close() # n = len(lines[6:]) # for i, line in enumerate(lines[6:]): # fields = line.split() # if verbose and i % ((n+10)/10) == 0: # log.critical('Processing row %d of %d' % (i, nrows)) # # if len(fields) != ncols: # msg = 'Wrong number of columns in file "%s" line %d\n' % (name_in, i) # msg += 'I got %d elements, but there should have been %d\n' % (len(fields), ncols) # raise Exception, msg # # elevation[i, :] = num.array([float(x) for x in fields]) fid.close()
def esri2sww(bath_dir, elevation_dir, ucur_dir, vcur_dir, sww_file, minlat=None, maxlat=None, minlon=None, maxlon=None, zscale=1, mean_stage=0, fail_on_NaN=True, elevation_NaN_filler=0, bath_prefix='ba', elevation_prefix='el', verbose=False): """ Produce an sww boundary file, from esri ascii data from CSIRO. Also convert latitude and longitude to UTM. All coordinates are assumed to be given in the GDA94 datum. assume: All files are in esri ascii format 4 types of information bathymetry elevation u velocity v velocity Assumptions The metadata of all the files is the same Each type is in a seperate directory One bath file with extention .000 The time period is less than 24hrs and uniform. """ from anuga.file.netcdf import NetCDFFile from anuga.coordinate_transforms.redfearn import redfearn if sww_file[-4:] != '.sww': raise IOError('Output file %s should be of type .sww.' % sww_file) # So if we want to change the precision it's done here precision = netcdf_float # go in to the bath dir and load the only file, bath_files = os.listdir(bath_dir) bath_file = bath_files[0] bath_dir_file = bath_dir + os.sep + bath_file bath_metadata, bath_grid = _read_asc(bath_dir_file) #Use the date.time of the bath file as a basis for #the start time for other files base_start = bath_file[-12:] #go into the elevation dir and load the 000 file elevation_dir_file = elevation_dir + os.sep + elevation_prefix \ + base_start elevation_files = os.listdir(elevation_dir) ucur_files = os.listdir(ucur_dir) vcur_files = os.listdir(vcur_dir) elevation_files.sort() # the first elevation file should be the # file with the same base name as the bath data assert elevation_files[0] == 'el' + base_start number_of_latitudes = bath_grid.shape[0] number_of_longitudes = bath_grid.shape[1] number_of_volumes = (number_of_latitudes - 1) * (number_of_longitudes - 1) * 2 longitudes = [bath_metadata['xllcorner'] + x*bath_metadata['cellsize'] \ for x in range(number_of_longitudes)] latitudes = [bath_metadata['yllcorner'] + y*bath_metadata['cellsize'] \ for y in range(number_of_latitudes)] # reverse order of lat, so the first lat represents the first grid row latitudes.reverse() kmin, kmax, lmin, lmax = get_min_max_indices(latitudes[:], longitudes[:], minlat=minlat, maxlat=maxlat, minlon=minlon, maxlon=maxlon) bath_grid = bath_grid[kmin:kmax, lmin:lmax] latitudes = latitudes[kmin:kmax] longitudes = longitudes[lmin:lmax] number_of_latitudes = len(latitudes) number_of_longitudes = len(longitudes) number_of_times = len(os.listdir(elevation_dir)) number_of_points = number_of_latitudes * number_of_longitudes number_of_volumes = (number_of_latitudes - 1) * (number_of_longitudes - 1) * 2 #Work out the times if len(elevation_files) > 1: # Assume: The time period is less than 24hrs. time_period = (int(elevation_files[1][-3:]) \ - int(elevation_files[0][-3:])) * 60*60 times = [x * time_period for x in range(len(elevation_files))] else: times = [0.0] if verbose: log.critical('------------------------------------------------') log.critical('Statistics:') log.critical(' Extent (lat/lon):') log.critical(' lat in [%f, %f], len(lat) == %d' % (min(latitudes), max(latitudes), len(latitudes))) log.critical(' lon in [%f, %f], len(lon) == %d' % (min(longitudes), max(longitudes), len(longitudes))) log.critical(' t in [%f, %f], len(t) == %d' % (min(times), max(times), len(times))) ######### WRITE THE SWW FILE ############# # NetCDF file definition outfile = NetCDFFile(sww_file, netcdf_mode_w) #Create new file outfile.institution = 'Geoscience Australia' outfile.description = 'Converted from XXX' #For sww compatibility outfile.smoothing = 'Yes' outfile.order = 1 #Start time in seconds since the epoch (midnight 1/1/1970) outfile.starttime = starttime = times[0] # dimension definitions outfile.createDimension('number_of_volumes', number_of_volumes) outfile.createDimension('number_of_vertices', 3) outfile.createDimension('number_of_points', number_of_points) outfile.createDimension('number_of_timesteps', number_of_times) # variable definitions outfile.createVariable('x', precision, ('number_of_points', )) outfile.createVariable('y', precision, ('number_of_points', )) outfile.createVariable('elevation', precision, ('number_of_points', )) #FIXME: Backwards compatibility #outfile.createVariable('z', precision, ('number_of_points',)) ################################# outfile.createVariable('volumes', netcdf_int, ('number_of_volumes', 'number_of_vertices')) outfile.createVariable('time', precision, ('number_of_timesteps', )) outfile.createVariable('stage', precision, ('number_of_timesteps', 'number_of_points')) outfile.createVariable('xmomentum', precision, ('number_of_timesteps', 'number_of_points')) outfile.createVariable('ymomentum', precision, ('number_of_timesteps', 'number_of_points')) #Store from anuga.coordinate_transforms.redfearn import redfearn x = num.zeros(number_of_points, num.float) #Easting y = num.zeros(number_of_points, num.float) #Northing if verbose: log.critical('Making triangular grid') #Get zone of 1st point. refzone, _, _ = redfearn(latitudes[0], longitudes[0]) vertices = {} i = 0 for k, lat in enumerate(latitudes): for l, lon in enumerate(longitudes): vertices[l, k] = i zone, easting, northing = redfearn(lat, lon) #msg = 'Zone boundary crossed at longitude =', lon #assert zone == refzone, msg #print '%7.2f %7.2f %8.2f %8.2f' %(lon, lat, easting, northing) x[i] = easting y[i] = northing i += 1 #Construct 2 triangles per 'rectangular' element volumes = [] for l in range(number_of_longitudes - 1): #X direction for k in range(number_of_latitudes - 1): #Y direction v1 = vertices[l, k + 1] v2 = vertices[l, k] v3 = vertices[l + 1, k + 1] v4 = vertices[l + 1, k] #Note, this is different to the ferrit2sww code #since the order of the lats is reversed. volumes.append([v1, v3, v2]) #Upper element volumes.append([v4, v2, v3]) #Lower element volumes = num.array(volumes, num.int) #array default# geo_ref = Geo_reference(refzone, min(x), min(y)) geo_ref.write_NetCDF(outfile) # This will put the geo ref in the middle #geo_ref = Geo_reference(refzone, (max(x)+min(x))/2., (max(x)+min(y))/2.) if verbose: log.critical('------------------------------------------------') log.critical('More Statistics:') log.critical(' Extent (/lon):') log.critical(' x in [%f, %f], len(lat) == %d' % (min(x), max(x), len(x))) log.critical(' y in [%f, %f], len(lon) == %d' % (min(y), max(y), len(y))) log.critical('geo_ref: ', geo_ref) z = num.resize(bath_grid, outfile.variables['elevation'][:].shape) outfile.variables['x'][:] = x - geo_ref.get_xllcorner() outfile.variables['y'][:] = y - geo_ref.get_yllcorner() # FIXME (Ole): Remove once viewer has been recompiled and changed # to use elevation instead of z #outfile.variables['z'][:] = z outfile.variables['elevation'][:] = z outfile.variables['volumes'][:] = volumes.astype( num.int32) # On Opteron 64 stage = outfile.variables['stage'] xmomentum = outfile.variables['xmomentum'] ymomentum = outfile.variables['ymomentum'] outfile.variables['time'][:] = times #Store time relative if verbose: log.critical('Converting quantities') n = number_of_times for j in range(number_of_times): # load in files elevation_meta, elevation_grid = \ _read_asc(elevation_dir + os.sep + elevation_files[j]) _, u_momentum_grid = _read_asc(ucur_dir + os.sep + ucur_files[j]) _, v_momentum_grid = _read_asc(vcur_dir + os.sep + vcur_files[j]) #cut matrix to desired size elevation_grid = elevation_grid[kmin:kmax, lmin:lmax] u_momentum_grid = u_momentum_grid[kmin:kmax, lmin:lmax] v_momentum_grid = v_momentum_grid[kmin:kmax, lmin:lmax] # handle missing values missing = (elevation_grid == elevation_meta['NODATA_value']) if num.sometrue(missing): if fail_on_NaN: msg = 'File %s contains missing values' \ % (elevation_files[j]) raise_(DataMissingValuesError, msg) else: elevation_grid = elevation_grid*(missing==0) \ + missing*elevation_NaN_filler if verbose and j % (old_div((n + 10), 10)) == 0: log.critical(' Doing %d of %d' % (j, n)) i = 0 for k in range(number_of_latitudes): #Y direction for l in range(number_of_longitudes): #X direction w = zscale * elevation_grid[k, l] + mean_stage stage[j, i] = w h = w - z[i] xmomentum[j, i] = u_momentum_grid[k, l] * h ymomentum[j, i] = v_momentum_grid[k, l] * h i += 1 outfile.close()
def _dem2pts(name_in, name_out=None, verbose=False, easting_min=None, easting_max=None, northing_min=None, northing_max=None): """Read Digitial Elevation model from the following NetCDF format (.dem) Internal function. See public function dem2pts for details. """ # FIXME: Can this be written feasibly using write_pts? import os from anuga.file.netcdf import NetCDFFile root = name_in[:-4] if name_in[-4:] == '.asc': intermediate = root + '.dem' if verbose: log.critical('Preconvert %s from asc to %s' % \ (name_in, intermediate)) asc2dem(name_in) name_in = intermediate elif name_in[-4:] != '.dem': raise IOError('Input file %s should be of type .asc or .dem.' % name_in) if name_out != None and basename_out[-4:] != '.pts': raise IOError('Input file %s should be of type .pts.' % name_out) # Get NetCDF infile = NetCDFFile(name_in, netcdf_mode_r) if verbose: log.critical('Reading DEM from %s' % (name_in)) ncols = int(infile.ncols) nrows = int(infile.nrows) xllcorner = float(infile.xllcorner) # Easting of lower left corner yllcorner = float(infile.yllcorner) # Northing of lower left corner cellsize = float(infile.cellsize) NODATA_value = float(infile.NODATA_value) dem_elevation = infile.variables['elevation'] zone = int(infile.zone) false_easting = float(infile.false_easting) false_northing = float(infile.false_northing) #print ncols, nrows, xllcorner,yllcorner, cellsize, NODATA_value, zone # Text strings projection = infile.projection datum = infile.datum units = infile.units #print projection, datum, units # Get output file if name_out is None: ptsname = root + '.pts' else: ptsname = name_out if verbose: log.critical('Store to NetCDF file %s' % ptsname) # NetCDF file definition outfile = NetCDFFile(ptsname, netcdf_mode_w) # Create new file outfile.institution = 'Geoscience Australia' outfile.description = 'NetCDF pts format for compact and portable ' \ 'storage of spatial point data' # Assign default values if easting_min is None: easting_min = xllcorner if easting_max is None: easting_max = xllcorner + ncols * cellsize if northing_min is None: northing_min = yllcorner if northing_max is None: northing_max = yllcorner + nrows * cellsize #print easting_min, easting_max, northing_min, northing_max # Compute offsets to update georeferencing easting_offset = xllcorner - easting_min northing_offset = yllcorner - northing_min # Georeferencing outfile.zone = zone outfile.xllcorner = easting_min # Easting of lower left corner outfile.yllcorner = northing_min # Northing of lower left corner outfile.false_easting = false_easting outfile.false_northing = false_northing outfile.projection = projection outfile.datum = datum outfile.units = units # Grid info (FIXME: probably not going to be used, but heck) outfile.ncols = ncols outfile.nrows = nrows #dem_elevation_r = num.reshape(dem_elevation, (nrows, ncols)) totalnopoints = nrows * ncols # #======================================================================= # # Calculating number of NODATA_values for each row in clipped region # # FIXME: use array operations to do faster # nn = 0 # k = 0 # i1_0 = 0 # j1_0 = 0 # thisj = 0 # thisi = 0 # for i in range(nrows): # y = (nrows-i-1)*cellsize + yllcorner # for j in range(ncols): # x = j*cellsize + xllcorner # if easting_min <= x <= easting_max \ # and northing_min <= y <= northing_max: # thisj = j # thisi = i # if dem_elevation_r[i,j] == NODATA_value: # nn += 1 # # if k == 0: # i1_0 = i # j1_0 = j # # k += 1 # # index1 = j1_0 # index2 = thisj # # # Dimension definitions # nrows_in_bounding_box = int(round((northing_max-northing_min)/cellsize)) # ncols_in_bounding_box = int(round((easting_max-easting_min)/cellsize)) # # clippednopoints = (thisi+1-i1_0)*(thisj+1-j1_0) # nopoints = clippednopoints-nn # # clipped_dem_elev = dem_elevation_r[i1_0:thisi+1,j1_0:thisj+1] # # if verbose: # log.critical('There are %d values in the elevation' % totalnopoints) # log.critical('There are %d values in the clipped elevation' # % clippednopoints) # log.critical('There are %d NODATA_values in the clipped elevation' % nn) # # outfile.createDimension('number_of_points', nopoints) # outfile.createDimension('number_of_dimensions', 2) #This is 2d data # # # Variable definitions # outfile.createVariable('points', netcdf_float, ('number_of_points', # 'number_of_dimensions')) # outfile.createVariable('elevation', netcdf_float, ('number_of_points',)) # # # Get handles to the variables # points = outfile.variables['points'] # elevation = outfile.variables['elevation'] # # # Number of points # N = points.shape[0] # # lenv = index2-index1+1 # # # Store data # global_index = 0 # # for i in range(nrows): # for i in range(i1_0, thisi+1, 1): # if verbose and i % ((nrows+10)/10) == 0: # log.critical('Processing row %d of %d' % (i, nrows)) # # lower_index = global_index # # v = dem_elevation_r[i,index1:index2+1] # no_NODATA = num.sum(v == NODATA_value) # if no_NODATA > 0: # newcols = lenv - no_NODATA # ncols_in_bounding_box - no_NODATA # else: # newcols = lenv # ncols_in_bounding_box # # telev = num.zeros(newcols, num.float) # tpoints = num.zeros((newcols, 2), num.float) # # local_index = 0 # # y = (nrows-i-1)*cellsize + yllcorner # #for j in range(ncols): # for j in range(j1_0,index2+1,1): # x = j*cellsize + xllcorner # if easting_min <= x <= easting_max \ # and northing_min <= y <= northing_max \ # and dem_elevation_r[i,j] != NODATA_value: # # #print [x-easting_min, y-northing_min] # #print x , y # #print easting_min, northing_min # #print xllcorner, yllcorner # #print cellsize # # tpoints[local_index, :] = [x-easting_min, y-northing_min] # telev[local_index] = dem_elevation_r[i, j] # global_index += 1 # local_index += 1 # # upper_index = global_index # # if upper_index == lower_index + newcols: # # # Seems to be an error with the windows version of # # Netcdf. The following gave errors # # try: # points[lower_index:upper_index, :] = tpoints # elevation[lower_index:upper_index] = telev # except: # # so used the following if an error occurs # for index in range(newcols): # points[index+lower_index, :] = tpoints[index,:] # elevation[index+lower_index] = telev[index] # # assert global_index == nopoints, 'index not equal to number of points' #======================================== # Do the preceeding with numpy #======================================== y = num.arange(nrows, dtype=num.float) y = yllcorner + (nrows - 1) * cellsize - y * cellsize x = num.arange(ncols, dtype=num.float) x = xllcorner + x * cellsize xx, yy = num.meshgrid(x, y) xx = xx.flatten() yy = yy.flatten() flag = num.logical_and( num.logical_and((xx <= easting_max), (xx >= easting_min)), num.logical_and((yy <= northing_max), (yy >= northing_min))) dem = dem_elevation[:].flatten() id = num.where(flag)[0] xx = xx[id] yy = yy[id] dem = dem[id] clippednopoints = len(dem) #print clippedpoints #print xx #print yy #print dem data_flag = dem != NODATA_value data_id = num.where(data_flag) xx = xx[data_id] yy = yy[data_id] dem = dem[data_id] nn = clippednopoints - len(dem) nopoints = len(dem) if verbose: log.critical('There are %d values in the elevation' % totalnopoints) log.critical('There are %d values in the clipped elevation' % clippednopoints) log.critical('There are %d NODATA_values in the clipped elevation' % nn) outfile.createDimension('number_of_points', nopoints) outfile.createDimension('number_of_dimensions', 2) #This is 2d data # Variable definitions outfile.createVariable('points', netcdf_float, ('number_of_points', 'number_of_dimensions')) outfile.createVariable('elevation', netcdf_float, ('number_of_points', )) # Get handles to the variables points = outfile.variables['points'] elevation = outfile.variables['elevation'] points[:, 0] = xx - easting_min points[:, 1] = yy - northing_min elevation[:] = dem infile.close() outfile.close()
def _dem2pts(name_in, name_out=None, verbose=False, easting_min=None, easting_max=None, northing_min=None, northing_max=None): """Read Digitial Elevation model from the following NetCDF format (.dem) Internal function. See public function dem2pts for details. """ # FIXME: Can this be written feasibly using write_pts? import os from anuga.file.netcdf import NetCDFFile root = name_in[:-4] if name_in[-4:] == '.asc': intermediate = root + '.dem' if verbose: log.critical('Preconvert %s from asc to %s' % \ (name_in, intermediate)) asc2dem(name_in) name_in = intermediate elif name_in[-4:] != '.dem': raise IOError('Input file %s should be of type .asc or .dem.' % name_in) if name_out != None and basename_out[-4:] != '.pts': raise IOError('Input file %s should be of type .pts.' % name_out) # Get NetCDF infile = NetCDFFile(name_in, netcdf_mode_r) if verbose: log.critical('Reading DEM from %s' % (name_in)) ncols = int(infile.ncols) nrows = int(infile.nrows) xllcorner = float(infile.xllcorner) # Easting of lower left corner yllcorner = float(infile.yllcorner) # Northing of lower left corner cellsize = float(infile.cellsize) NODATA_value = float(infile.NODATA_value) dem_elevation = infile.variables['elevation'] zone = int(infile.zone) false_easting = float(infile.false_easting) false_northing = float(infile.false_northing) #print ncols, nrows, xllcorner,yllcorner, cellsize, NODATA_value, zone # Text strings projection = infile.projection datum = infile.datum units = infile.units #print projection, datum, units # Get output file if name_out == None: ptsname = root + '.pts' else: ptsname = name_out if verbose: log.critical('Store to NetCDF file %s' % ptsname) # NetCDF file definition outfile = NetCDFFile(ptsname, netcdf_mode_w) # Create new file outfile.institution = 'Geoscience Australia' outfile.description = 'NetCDF pts format for compact and portable ' \ 'storage of spatial point data' # Assign default values if easting_min is None: easting_min = xllcorner if easting_max is None: easting_max = xllcorner + ncols*cellsize if northing_min is None: northing_min = yllcorner if northing_max is None: northing_max = yllcorner + nrows*cellsize #print easting_min, easting_max, northing_min, northing_max # Compute offsets to update georeferencing easting_offset = xllcorner - easting_min northing_offset = yllcorner - northing_min # Georeferencing outfile.zone = zone outfile.xllcorner = easting_min # Easting of lower left corner outfile.yllcorner = northing_min # Northing of lower left corner outfile.false_easting = false_easting outfile.false_northing = false_northing outfile.projection = projection outfile.datum = datum outfile.units = units # Grid info (FIXME: probably not going to be used, but heck) outfile.ncols = ncols outfile.nrows = nrows #dem_elevation_r = num.reshape(dem_elevation, (nrows, ncols)) totalnopoints = nrows*ncols # #======================================================================= # # Calculating number of NODATA_values for each row in clipped region # # FIXME: use array operations to do faster # nn = 0 # k = 0 # i1_0 = 0 # j1_0 = 0 # thisj = 0 # thisi = 0 # for i in range(nrows): # y = (nrows-i-1)*cellsize + yllcorner # for j in range(ncols): # x = j*cellsize + xllcorner # if easting_min <= x <= easting_max \ # and northing_min <= y <= northing_max: # thisj = j # thisi = i # if dem_elevation_r[i,j] == NODATA_value: # nn += 1 # # if k == 0: # i1_0 = i # j1_0 = j # # k += 1 # # index1 = j1_0 # index2 = thisj # # # Dimension definitions # nrows_in_bounding_box = int(round((northing_max-northing_min)/cellsize)) # ncols_in_bounding_box = int(round((easting_max-easting_min)/cellsize)) # # clippednopoints = (thisi+1-i1_0)*(thisj+1-j1_0) # nopoints = clippednopoints-nn # # clipped_dem_elev = dem_elevation_r[i1_0:thisi+1,j1_0:thisj+1] # # if verbose: # log.critical('There are %d values in the elevation' % totalnopoints) # log.critical('There are %d values in the clipped elevation' # % clippednopoints) # log.critical('There are %d NODATA_values in the clipped elevation' % nn) # # outfile.createDimension('number_of_points', nopoints) # outfile.createDimension('number_of_dimensions', 2) #This is 2d data # # # Variable definitions # outfile.createVariable('points', netcdf_float, ('number_of_points', # 'number_of_dimensions')) # outfile.createVariable('elevation', netcdf_float, ('number_of_points',)) # # # Get handles to the variables # points = outfile.variables['points'] # elevation = outfile.variables['elevation'] # # # Number of points # N = points.shape[0] # # lenv = index2-index1+1 # # # Store data # global_index = 0 # # for i in range(nrows): # for i in range(i1_0, thisi+1, 1): # if verbose and i % ((nrows+10)/10) == 0: # log.critical('Processing row %d of %d' % (i, nrows)) # # lower_index = global_index # # v = dem_elevation_r[i,index1:index2+1] # no_NODATA = num.sum(v == NODATA_value) # if no_NODATA > 0: # newcols = lenv - no_NODATA # ncols_in_bounding_box - no_NODATA # else: # newcols = lenv # ncols_in_bounding_box # # telev = num.zeros(newcols, num.float) # tpoints = num.zeros((newcols, 2), num.float) # # local_index = 0 # # y = (nrows-i-1)*cellsize + yllcorner # #for j in range(ncols): # for j in range(j1_0,index2+1,1): # x = j*cellsize + xllcorner # if easting_min <= x <= easting_max \ # and northing_min <= y <= northing_max \ # and dem_elevation_r[i,j] != NODATA_value: # # #print [x-easting_min, y-northing_min] # #print x , y # #print easting_min, northing_min # #print xllcorner, yllcorner # #print cellsize # # tpoints[local_index, :] = [x-easting_min, y-northing_min] # telev[local_index] = dem_elevation_r[i, j] # global_index += 1 # local_index += 1 # # upper_index = global_index # # if upper_index == lower_index + newcols: # # # Seems to be an error with the windows version of # # Netcdf. The following gave errors # # try: # points[lower_index:upper_index, :] = tpoints # elevation[lower_index:upper_index] = telev # except: # # so used the following if an error occurs # for index in range(newcols): # points[index+lower_index, :] = tpoints[index,:] # elevation[index+lower_index] = telev[index] # # assert global_index == nopoints, 'index not equal to number of points' #======================================== # Do the preceeding with numpy #======================================== y = num.arange(nrows,dtype=num.float) y = yllcorner + (nrows-1)*cellsize - y*cellsize x = num.arange(ncols,dtype=num.float) x = xllcorner + x*cellsize xx,yy = num.meshgrid(x,y) xx = xx.flatten() yy = yy.flatten() flag = num.logical_and(num.logical_and((xx <= easting_max),(xx >= easting_min)), num.logical_and((yy <= northing_max),(yy >= northing_min))) dem = dem_elevation[:].flatten() id = num.where(flag)[0] xx = xx[id] yy = yy[id] dem = dem[id] clippednopoints = len(dem) #print clippedpoints #print xx #print yy #print dem data_flag = dem != NODATA_value data_id = num.where(data_flag) xx = xx[data_id] yy = yy[data_id] dem = dem[data_id] nn = clippednopoints - len(dem) nopoints = len(dem) if verbose: log.critical('There are %d values in the elevation' % totalnopoints) log.critical('There are %d values in the clipped elevation' % clippednopoints) log.critical('There are %d NODATA_values in the clipped elevation' % nn) outfile.createDimension('number_of_points', nopoints) outfile.createDimension('number_of_dimensions', 2) #This is 2d data # Variable definitions outfile.createVariable('points', netcdf_float, ('number_of_points', 'number_of_dimensions')) outfile.createVariable('elevation', netcdf_float, ('number_of_points',)) # Get handles to the variables points = outfile.variables['points'] elevation = outfile.variables['elevation'] points[:,0] = xx - easting_min points[:,1] = yy - northing_min elevation[:] = dem infile.close() outfile.close()
def _convert_dem_from_llasc2pts(name_in, name_out = None, show_progress=False, verbose=False,): """Read Digital Elevation model from the following LL ASCII format (.asc) Internal function. See public function convert_dem_from_ascii2netcdf for details. """ import os from anuga.file.netcdf import NetCDFFile #Read DEM data datafile = open(name_in) if verbose: log.critical('Reading DEM from %s' % (name_in)) lines = datafile.readlines() datafile.close() if verbose: log.critical('Got %d lines' % len(lines)) ncols = int(lines[0].split()[1].strip()) nrows = int(lines[1].split()[1].strip()) # Do cellsize (line 4) before line 2 and 3 cellsize = float(lines[4].split()[1].strip()) xref = lines[2].split() if xref[0].strip() == 'xllcorner': xllcorner = float(xref[1].strip()) elif xref[0].strip() == 'xllcenter': xllcorner = float(xref[1].strip()) # - 0.5*cellsize # Correct offset else: msg = 'Unknown keyword: %s' % xref[0].strip() raise_(Exception, msg) yref = lines[3].split() if yref[0].strip() == 'yllcorner': yllcorner = float(yref[1].strip()) elif yref[0].strip() == 'yllcenter': yllcorner = float(yref[1].strip()) # - 0.5*cellsize # Correct offset else: msg = 'Unknown keyword: %s' % yref[0].strip() raise_(Exception, msg) NODATA_value = float(lines[5].split()[1].strip()) assert len(lines) == nrows + 6 dem_elevation = num.loadtxt(lines, skiprows=6, dtype=float) totalnopoints = nrows*ncols y = num.arange(nrows,dtype=num.float) y = yllcorner + (nrows-1)*cellsize - y*cellsize x = num.arange(ncols,dtype=num.float) x = xllcorner + x*cellsize #print(xllcorner) #print(x) #print(yllcorner) #print(y) xx,yy = num.meshgrid(x,y) xx = xx.flatten() yy = yy.flatten() dem = dem_elevation[:].flatten() # ==================== # remove NODATA points # ==================== data_flag = dem != NODATA_value data_id = num.where(data_flag) xx = xx[data_id] yy = yy[data_id] dem = dem[data_id] nn = totalnopoints - len(dem) nopoints = len(dem) # ===================================== # Convert xx and yy to UTM # ===================================== points_UTM, zone = convert_from_latlon_to_utm(latitudes=yy, longitudes=xx, show_progress=show_progress) points_UTM = num.asarray(points_UTM, dtype=float) corners, zone_c = convert_from_latlon_to_utm(latitudes=yllcorner, longitudes=xllcorner) xllcorner = corners[0][0] yllcorner = corners[0][1] assert zone == zone_c points_UTM = points_UTM - corners # =============================== # Step up for writing to pts file # =============================== if name_out is None: netcdfname = name_in[:-4]+'.pts' else: netcdfname = name_out + '.pts' if verbose: log.critical('Store to NetCDF file %s' % netcdfname) # NetCDF file definition outfile = NetCDFFile(netcdfname, netcdf_mode_w) # Create new file outfile.institution = 'Geoscience Australia' outfile.description = 'NetCDF pts format for compact and portable ' \ 'storage of spatial point data' # Georeferencing outfile.zone = zone outfile.xllcorner = xllcorner # Easting of lower left corner outfile.yllcorner = yllcorner # Northing of lower left corner # Default settings outfile.false_easting = 500000.0 outfile.false_northing = 10000000.0 outfile.projection = 'UTM' outfile.datum = 'WGS84' outfile.units = 'METERS' # Grid info (FIXME: probably not going to be used, but heck) outfile.ncols = ncols outfile.nrows = nrows if verbose: log.critical('There are %d values in the elevation' % totalnopoints) log.critical('There are %d NODATA_values in the clipped elevation' % nn) outfile.createDimension('number_of_points', nopoints) outfile.createDimension('number_of_dimensions', 2) #This is 2d data # Variable definitions outfile.createVariable('points', netcdf_float, ('number_of_points', 'number_of_dimensions')) outfile.createVariable('elevation', netcdf_float, ('number_of_points',)) # Get handles to the variables points = outfile.variables['points'] elevation = outfile.variables['elevation'] points[:,:]= points_UTM elevation[:] = dem outfile.close()
def dem2dem(name_in, stencil, cellsize_new, name_out=None, verbose=False): """Read Digitial Elevation model from the following NetCDF format (.dem) Example: ncols 3121 nrows 1800 xllcorner 722000 yllcorner 5893000 cellsize 25 NODATA_value -9999 138.3698 137.4194 136.5062 135.5558 .......... Decimate data to cellsize_new using stencil and write to NetCDF dem format. """ import os from anuga.file.netcdf import NetCDFFile if name_in[-4:] != '.dem': raise IOError('Input file %s should be of type .dem.' % name_in) if name_out != None and basename_out[-4:] != '.dem': raise IOError('Input file %s should be of type .dem.' % name_out) #Open existing netcdf file to read infile = NetCDFFile(name_in, netcdf_mode_r) if verbose: log.critical('Reading DEM from %s' % inname) # Read metadata (convert from numpy.int32 to int where appropriate) ncols = int(infile.ncols) nrows = int(infile.nrows) xllcorner = infile.xllcorner yllcorner = infile.yllcorner cellsize = int(infile.cellsize) NODATA_value = int(infile.NODATA_value) zone = int(infile.zone) false_easting = infile.false_easting false_northing = infile.false_northing projection = infile.projection datum = infile.datum units = infile.units dem_elevation = infile.variables['elevation'] #Get output file name if name_out == None: outname = name_in[:-4] + '_' + repr(cellsize_new) + '.dem' else: outname = name_out if verbose: log.critical('Write decimated NetCDF file to %s' % outname) #Determine some dimensions for decimated grid (nrows_stencil, ncols_stencil) = stencil.shape x_offset = ncols_stencil / 2 y_offset = nrows_stencil / 2 cellsize_ratio = int(cellsize_new / cellsize) ncols_new = 1 + (ncols - ncols_stencil) / cellsize_ratio nrows_new = 1 + (nrows - nrows_stencil) / cellsize_ratio #print type(ncols_new), ncols_new #Open netcdf file for output outfile = NetCDFFile(outname, netcdf_mode_w) #Create new file outfile.institution = 'Geoscience Australia' outfile.description = 'NetCDF DEM format for compact and portable ' \ 'storage of spatial point data' #Georeferencing outfile.zone = zone outfile.projection = projection outfile.datum = datum outfile.units = units outfile.cellsize = cellsize_new outfile.NODATA_value = NODATA_value outfile.false_easting = false_easting outfile.false_northing = false_northing outfile.xllcorner = xllcorner + (x_offset * cellsize) outfile.yllcorner = yllcorner + (y_offset * cellsize) outfile.ncols = ncols_new outfile.nrows = nrows_new # dimension definition #print nrows_new, ncols_new, nrows_new*ncols_new #print type(nrows_new), type(ncols_new), type(nrows_new*ncols_new) outfile.createDimension('number_of_points', nrows_new * ncols_new) # variable definition outfile.createVariable('elevation', netcdf_float, ('number_of_points', )) # Get handle to the variable elevation = outfile.variables['elevation'] dem_elevation_r = num.reshape(dem_elevation, (nrows, ncols)) #Store data global_index = 0 for i in range(nrows_new): if verbose: log.critical('Processing row %d of %d' % (i, nrows_new)) lower_index = global_index telev = num.zeros(ncols_new, num.float) local_index = 0 trow = i * cellsize_ratio for j in range(ncols_new): tcol = j * cellsize_ratio tmp = dem_elevation_r[trow:trow + nrows_stencil, tcol:tcol + ncols_stencil] #if dem contains 1 or more NODATA_values set value in #decimated dem to NODATA_value, else compute decimated #value using stencil if num.sum(num.sum(num.equal(tmp, NODATA_value))) > 0: telev[local_index] = NODATA_value else: telev[local_index] = num.sum(num.sum(tmp * stencil)) global_index += 1 local_index += 1 upper_index = global_index elevation[lower_index:upper_index] = telev assert global_index == nrows_new*ncols_new, \ 'index not equal to number of points' infile.close() outfile.close()
def _generic_dem2pts(name_in, name_out=None, quantity_name=None, verbose=False, easting_min=None, easting_max=None, northing_min=None, northing_max=None): """Read raster from the following NetCDF format (.dem) Internal function. See public function generic_dem2pts for details. """ # FIXME: Can this be written feasibly using write_pts? import os from anuga.file.netcdf import NetCDFFile root = name_in[:-4] if name_in[-4:] == '.asc': intermediate = root + '.dem' if verbose: log.critical('Preconvert %s from asc to %s' % \ (name_in, intermediate)) asc2dem(name_in) name_in = intermediate elif name_in[-4:] != '.dem': raise IOError('Input file %s should be of type .asc or .dem.' % name_in) if name_out != None and basename_out[-4:] != '.pts': raise IOError('Input file %s should be of type .pts.' % name_out) # Get NetCDF infile = NetCDFFile(name_in, netcdf_mode_r) if verbose: log.critical('Reading raster from %s' % (name_in)) ncols = int(infile.ncols) nrows = int(infile.nrows) xllcorner = float(infile.xllcorner) # Easting of lower left corner yllcorner = float(infile.yllcorner) # Northing of lower left corner cellsize = float(infile.cellsize) NODATA_value = float(infile.NODATA_value) dem_elevation = infile.variables[quantity_name] zone = int(infile.zone) false_easting = float(infile.false_easting) false_northing = float(infile.false_northing) #print ncols, nrows, xllcorner,yllcorner, cellsize, NODATA_value, zone # Text strings projection = infile.projection datum = infile.datum units = infile.units #print projection, datum, units # Get output file if name_out == None: ptsname = root + '.pts' else: ptsname = name_out if verbose: log.critical('Store to NetCDF file %s' % ptsname) # NetCDF file definition outfile = NetCDFFile(ptsname, netcdf_mode_w) # Create new file outfile.institution = 'Geoscience Australia' outfile.description = 'NetCDF pts format for compact and portable ' \ 'storage of spatial point data' # Assign default values if easting_min is None: easting_min = xllcorner if easting_max is None: easting_max = xllcorner + ncols * cellsize if northing_min is None: northing_min = yllcorner if northing_max is None: northing_max = yllcorner + nrows * cellsize #print easting_min, easting_max, northing_min, northing_max # Compute offsets to update georeferencing easting_offset = xllcorner - easting_min northing_offset = yllcorner - northing_min # Georeferencing outfile.zone = zone outfile.xllcorner = easting_min # Easting of lower left corner outfile.yllcorner = northing_min # Northing of lower left corner outfile.false_easting = false_easting outfile.false_northing = false_northing outfile.projection = projection outfile.datum = datum outfile.units = units # Grid info (FIXME: probably not going to be used, but heck) outfile.ncols = ncols outfile.nrows = nrows dem_elevation_r = num.reshape(dem_elevation, (nrows, ncols)) totalnopoints = nrows * ncols #======================================== # Do the preceeding with numpy #======================================== y = num.arange(nrows, dtype=num.float) y = yllcorner + (nrows - 1) * cellsize - y * cellsize x = num.arange(ncols, dtype=num.float) x = xllcorner + x * cellsize xx, yy = num.meshgrid(x, y) xx = xx.flatten() yy = yy.flatten() flag = num.logical_and( num.logical_and((xx <= easting_max), (xx >= easting_min)), num.logical_and((yy <= northing_max), (yy >= northing_min))) dem = dem_elevation[:].flatten() id = num.where(flag)[0] xx = xx[id] yy = yy[id] dem = dem[id] clippednopoints = len(dem) #print clippedpoints #print xx #print yy #print dem data_flag = dem != NODATA_value data_id = num.where(data_flag) xx = xx[data_id] yy = yy[data_id] dem = dem[data_id] nn = clippednopoints - len(dem) nopoints = len(dem) if verbose: log.critical('There are %d values in the raster' % totalnopoints) log.critical('There are %d values in the clipped raster' % clippednopoints) log.critical('There are %d NODATA_values in the clipped raster' % nn) outfile.createDimension('number_of_points', nopoints) outfile.createDimension('number_of_dimensions', 2) #This is 2d data # Variable definitions outfile.createVariable('points', netcdf_float, ('number_of_points', 'number_of_dimensions')) outfile.createVariable(quantity_name, netcdf_float, ('number_of_points', )) # Get handles to the variables points = outfile.variables['points'] elevation = outfile.variables[quantity_name] points[:, 0] = xx - easting_min points[:, 1] = yy - northing_min elevation[:] = dem infile.close() outfile.close()
def dem2dem(name_in, stencil, cellsize_new, name_out=None, verbose=False): """Read Digitial Elevation model from the following NetCDF format (.dem) Example: ncols 3121 nrows 1800 xllcorner 722000 yllcorner 5893000 cellsize 25 NODATA_value -9999 138.3698 137.4194 136.5062 135.5558 .......... Decimate data to cellsize_new using stencil and write to NetCDF dem format. """ import os from anuga.file.netcdf import NetCDFFile if name_in[-4:] != '.dem': raise IOError('Input file %s should be of type .dem.' % name_in) if name_out != None and basename_out[-4:] != '.dem': raise IOError('Input file %s should be of type .dem.' % name_out) #Open existing netcdf file to read infile = NetCDFFile(name_in, netcdf_mode_r) if verbose: log.critical('Reading DEM from %s' % inname) # Read metadata (convert from numpy.int32 to int where appropriate) ncols = int(infile.ncols) nrows = int(infile.nrows) xllcorner = infile.xllcorner yllcorner = infile.yllcorner cellsize = int(infile.cellsize) NODATA_value = int(infile.NODATA_value) zone = int(infile.zone) false_easting = infile.false_easting false_northing = infile.false_northing projection = infile.projection datum = infile.datum units = infile.units dem_elevation = infile.variables['elevation'] #Get output file name if name_out == None: outname = name_in[:-4] + '_' + repr(cellsize_new) + '.dem' else: outname = name_out if verbose: log.critical('Write decimated NetCDF file to %s' % outname) #Determine some dimensions for decimated grid (nrows_stencil, ncols_stencil) = stencil.shape x_offset = ncols_stencil / 2 y_offset = nrows_stencil / 2 cellsize_ratio = int(cellsize_new / cellsize) ncols_new = 1 + (ncols - ncols_stencil) / cellsize_ratio nrows_new = 1 + (nrows - nrows_stencil) / cellsize_ratio #print type(ncols_new), ncols_new #Open netcdf file for output outfile = NetCDFFile(outname, netcdf_mode_w) #Create new file outfile.institution = 'Geoscience Australia' outfile.description = 'NetCDF DEM format for compact and portable ' \ 'storage of spatial point data' #Georeferencing outfile.zone = zone outfile.projection = projection outfile.datum = datum outfile.units = units outfile.cellsize = cellsize_new outfile.NODATA_value = NODATA_value outfile.false_easting = false_easting outfile.false_northing = false_northing outfile.xllcorner = xllcorner + (x_offset * cellsize) outfile.yllcorner = yllcorner + (y_offset * cellsize) outfile.ncols = ncols_new outfile.nrows = nrows_new # dimension definition #print nrows_new, ncols_new, nrows_new*ncols_new #print type(nrows_new), type(ncols_new), type(nrows_new*ncols_new) outfile.createDimension('number_of_points', nrows_new*ncols_new) # variable definition outfile.createVariable('elevation', netcdf_float, ('number_of_points',)) # Get handle to the variable elevation = outfile.variables['elevation'] dem_elevation_r = num.reshape(dem_elevation, (nrows, ncols)) #Store data global_index = 0 for i in range(nrows_new): if verbose: log.critical('Processing row %d of %d' % (i, nrows_new)) lower_index = global_index telev = num.zeros(ncols_new, num.float) local_index = 0 trow = i * cellsize_ratio for j in range(ncols_new): tcol = j * cellsize_ratio tmp = dem_elevation_r[trow:trow+nrows_stencil, tcol:tcol+ncols_stencil] #if dem contains 1 or more NODATA_values set value in #decimated dem to NODATA_value, else compute decimated #value using stencil if num.sum(num.sum(num.equal(tmp, NODATA_value))) > 0: telev[local_index] = NODATA_value else: telev[local_index] = num.sum(num.sum(tmp * stencil)) global_index += 1 local_index += 1 upper_index = global_index elevation[lower_index:upper_index] = telev assert global_index == nrows_new*ncols_new, \ 'index not equal to number of points' infile.close() outfile.close()
def timefile2netcdf(file_text, file_out = None, quantity_names=None, \ time_as_seconds=False): """Template for converting typical text files with time series to NetCDF tms file. The file format is assumed to be either two fields separated by a comma: time [DD/MM/YY hh:mm:ss], value0 value1 value2 ... E.g 31/08/04 00:00:00, 1.328223 0 0 31/08/04 00:15:00, 1.292912 0 0 or time (seconds), value0 value1 value2 ... 0.0, 1.328223 0 0 0.1, 1.292912 0 0 will provide a time dependent function f(t) with three attributes filename is assumed to be the rootname with extensions .txt/.tms and .sww """ import time, calendar from anuga.config import time_format from anuga.utilities.numerical_tools import ensure_numeric if file_text[-4:] != '.txt': raise IOError('Input file %s should be of type .txt.' % file_text) if file_out is None: file_out = file_text[:-4] + '.tms' fid = open(file_text) line = fid.readline() fid.close() fields = line.split(',') msg = "File %s must have the format 'datetime, value0 value1 value2 ...'" \ % file_text assert len(fields) == 2, msg if not time_as_seconds: try: starttime = calendar.timegm(time.strptime(fields[0], time_format)) except ValueError: msg = 'First field in file %s must be' % file_text msg += ' date-time with format %s.\n' % time_format msg += 'I got %s instead.' % fields[0] raise DataTimeError, msg else: try: starttime = float(fields[0]) except Exception: msg = "Bad time format" raise DataTimeError, msg # Split values values = [] for value in fields[1].split(): values.append(float(value)) q = ensure_numeric(values) msg = 'ERROR: File must contain at least one independent value' assert len(q.shape) == 1, msg # Read times proper from anuga.config import time_format import time, calendar fid = open(file_text) lines = fid.readlines() fid.close() N = len(lines) d = len(q) T = num.zeros(N, num.float) # Time Q = num.zeros((N, d), num.float) # Values for i, line in enumerate(lines): fields = line.split(',') if not time_as_seconds: realtime = calendar.timegm(time.strptime(fields[0], time_format)) else: realtime = float(fields[0]) T[i] = realtime - starttime for j, value in enumerate(fields[1].split()): Q[i, j] = float(value) msg = 'File %s must list time as a monotonuosly ' % file_text msg += 'increasing sequence' assert num.alltrue(T[1:] - T[:-1] > 0), msg #Create NetCDF file fid = NetCDFFile(file_out, netcdf_mode_w) fid.institution = 'Geoscience Australia' fid.description = 'Time series' #Reference point #Start time in seconds since the epoch (midnight 1/1/1970) #FIXME: Use Georef fid.starttime = starttime # dimension definitions #fid.createDimension('number_of_volumes', self.number_of_volumes) #fid.createDimension('number_of_vertices', 3) fid.createDimension('number_of_timesteps', len(T)) fid.createVariable('time', netcdf_float, ('number_of_timesteps', )) fid.variables['time'][:] = T for i in range(Q.shape[1]): try: name = quantity_names[i] except: name = 'Attribute%d' % i fid.createVariable(name, netcdf_float, ('number_of_timesteps', )) fid.variables[name][:] = Q[:, i] fid.close()
def _write_msh_file(file_name, mesh): """Write .msh NetCDF file WARNING: This function mangles the mesh data structure """ # FIXME(Ole and John): We ran into a problem on Bogong (64 bit) # where integers appeared as arrays. This may be similar to # problem seen by Steve in changeset:2778 where he had to wrap # them in int. Now we are trying with the native Integer format # (Int == 'l' == Int64). However, that caused casting errors, when # 64bit arrays are to be assigned to their NetCDF counterparts. It # seems that the NetCDF arrays are 32bit even though they are # created with the type Int64. Need to look at the NetCDF library # in more detail. IntType = num.int32 #IntType = Int #print 'mesh vertices',mesh['vertices'].shape #the triangulation mesh['vertices'] = num.array(mesh['vertices'], num.float) mesh['vertex_attribute_titles'] = \ num.array(string_to_char(mesh['vertex_attribute_titles']), num.character) num_attributes = len(mesh['vertex_attribute_titles']) num_vertices = mesh['vertices'].shape[0] #print 'num_attrib ',num_attributes if mesh['vertex_attributes'] != None: mesh['vertex_attributes'] = \ num.array(mesh['vertex_attributes'], num.float) if num_attributes > 0 : mesh['vertex_attributes'] = \ num.reshape(mesh['vertex_attributes'],(num_vertices,-1)) mesh['segments'] = num.array(mesh['segments'], IntType) mesh['segment_tags'] = num.array(string_to_char(mesh['segment_tags']), num.character) mesh['triangles'] = num.array(mesh['triangles'], IntType) mesh['triangle_tags'] = num.array(string_to_char(mesh['triangle_tags']), num.character) mesh['triangle_neighbors'] = \ num.array(mesh['triangle_neighbors'], IntType) #the outline mesh['points'] = num.array(mesh['points'], num.float) mesh['point_attributes'] = num.array(mesh['point_attributes'], num.float) mesh['outline_segments'] = num.array(mesh['outline_segments'], IntType) mesh['outline_segment_tags'] = \ num.array(string_to_char(mesh['outline_segment_tags']), num.character) mesh['holes'] = num.array(mesh['holes'], num.float) mesh['regions'] = num.array(mesh['regions'], num.float) mesh['region_tags'] = num.array(string_to_char(mesh['region_tags']), num.character) mesh['region_max_areas'] = num.array(mesh['region_max_areas'], num.float) # NetCDF file definition try: outfile = NetCDFFile(file_name, netcdf_mode_w) except IOError: msg = 'File %s could not be created' % file_name raise Exception, msg #Create new file outfile.institution = 'Geoscience Australia' outfile.description = 'NetCDF format for compact and portable storage ' + \ 'of spatial point data' # dimension definitions - fixed outfile.createDimension('num_of_dimensions', 2) # This is 2d data outfile.createDimension('num_of_segment_ends', 2) # Segs have two points outfile.createDimension('num_of_triangle_vertices', 3) outfile.createDimension('num_of_triangle_faces', 3) outfile.createDimension('num_of_region_max_area', 1) # Create dimensions, variables and set the variables # trianglulation # vertices if (mesh['vertices'].shape[0] > 0): outfile.createDimension('num_of_vertices', mesh['vertices'].shape[0]) outfile.createVariable('vertices', netcdf_float, ('num_of_vertices', 'num_of_dimensions')) outfile.variables['vertices'][:] = mesh['vertices'] #print 'mesh vertex attributes', mesh['vertex_attributes'].shape if (mesh['vertex_attributes'] is not None and (mesh['vertex_attributes'].shape[0] > 0 and mesh['vertex_attributes'].shape[1] > 0)): outfile.createDimension('num_of_vertex_attributes', mesh['vertex_attributes'].shape[1]) outfile.createDimension('num_of_vertex_attribute_title_chars', mesh['vertex_attribute_titles'].shape[1]) outfile.createVariable('vertex_attributes', netcdf_float, ('num_of_vertices', 'num_of_vertex_attributes')) outfile.createVariable('vertex_attribute_titles', netcdf_char, ('num_of_vertex_attributes', 'num_of_vertex_attribute_title_chars')) outfile.variables['vertex_attributes'][:] = \ mesh['vertex_attributes'] outfile.variables['vertex_attribute_titles'][:] = \ mesh['vertex_attribute_titles'] # segments if (mesh['segments'].shape[0] > 0): outfile.createDimension('num_of_segments', mesh['segments'].shape[0]) outfile.createVariable('segments', netcdf_int, ('num_of_segments', 'num_of_segment_ends')) outfile.variables['segments'][:] = mesh['segments'] if (mesh['segment_tags'].shape[1] > 0): outfile.createDimension('num_of_segment_tag_chars', mesh['segment_tags'].shape[1]) outfile.createVariable('segment_tags', netcdf_char, ('num_of_segments', 'num_of_segment_tag_chars')) outfile.variables['segment_tags'][:] = mesh['segment_tags'] # triangles if (mesh['triangles'].shape[0] > 0): outfile.createDimension('num_of_triangles', mesh['triangles'].shape[0]) outfile.createVariable('triangles', netcdf_int, ('num_of_triangles', 'num_of_triangle_vertices')) outfile.createVariable('triangle_neighbors', netcdf_int, ('num_of_triangles', 'num_of_triangle_faces')) outfile.variables['triangles'][:] = mesh['triangles'] outfile.variables['triangle_neighbors'][:] = mesh['triangle_neighbors'] if (mesh['triangle_tags'] is not None and (mesh['triangle_tags'].shape[1] > 0)): outfile.createDimension('num_of_triangle_tag_chars', mesh['triangle_tags'].shape[1]) outfile.createVariable('triangle_tags', netcdf_char, ('num_of_triangles', 'num_of_triangle_tag_chars')) outfile.variables['triangle_tags'][:] = mesh['triangle_tags'] # outline # points if (mesh['points'].shape[0] > 0): outfile.createDimension('num_of_points', mesh['points'].shape[0]) outfile.createVariable('points', netcdf_float, ('num_of_points', 'num_of_dimensions')) outfile.variables['points'][:] = mesh['points'] if mesh['point_attributes'].shape[0] > 0 \ and mesh['point_attributes'].shape[1] > 0: outfile.createDimension('num_of_point_attributes', mesh['point_attributes'].shape[1]) outfile.createVariable('point_attributes', netcdf_float, ('num_of_points', 'num_of_point_attributes')) outfile.variables['point_attributes'][:] = mesh['point_attributes'] # outline_segments if mesh['outline_segments'].shape[0] > 0: outfile.createDimension('num_of_outline_segments', mesh['outline_segments'].shape[0]) outfile.createVariable('outline_segments', netcdf_int, ('num_of_outline_segments', 'num_of_segment_ends')) outfile.variables['outline_segments'][:] = mesh['outline_segments'] if mesh['outline_segment_tags'].shape[1] > 0: outfile.createDimension('num_of_outline_segment_tag_chars', mesh['outline_segment_tags'].shape[1]) outfile.createVariable('outline_segment_tags', netcdf_char, ('num_of_outline_segments', 'num_of_outline_segment_tag_chars')) outfile.variables['outline_segment_tags'][:] = \ mesh['outline_segment_tags'] # holes if (mesh['holes'].shape[0] > 0): outfile.createDimension('num_of_holes', mesh['holes'].shape[0]) outfile.createVariable('holes', netcdf_float, ('num_of_holes', 'num_of_dimensions')) outfile.variables['holes'][:] = mesh['holes'] # regions if (mesh['regions'].shape[0] > 0): outfile.createDimension('num_of_regions', mesh['regions'].shape[0]) outfile.createVariable('regions', netcdf_float, ('num_of_regions', 'num_of_dimensions')) outfile.createVariable('region_max_areas', netcdf_float, ('num_of_regions',)) outfile.variables['regions'][:] = mesh['regions'] outfile.variables['region_max_areas'][:] = mesh['region_max_areas'] if (mesh['region_tags'].shape[1] > 0): outfile.createDimension('num_of_region_tag_chars', mesh['region_tags'].shape[1]) outfile.createVariable('region_tags', netcdf_char, ('num_of_regions', 'num_of_region_tag_chars')) outfile.variables['region_tags'][:] = mesh['region_tags'] # geo_reference info if mesh.has_key('geo_reference') and not mesh['geo_reference'] == None: mesh['geo_reference'].write_NetCDF(outfile) outfile.close()
def timefile2netcdf(file_text, file_out = None, quantity_names=None, \ time_as_seconds=False): """Template for converting typical text files with time series to NetCDF tms file. The file format is assumed to be either two fields separated by a comma: time [DD/MM/YY hh:mm:ss], value0 value1 value2 ... E.g 31/08/04 00:00:00, 1.328223 0 0 31/08/04 00:15:00, 1.292912 0 0 or time (seconds), value0 value1 value2 ... 0.0, 1.328223 0 0 0.1, 1.292912 0 0 will provide a time dependent function f(t) with three attributes filename is assumed to be the rootname with extensions .txt/.tms and .sww """ import time, calendar from anuga.config import time_format from anuga.utilities.numerical_tools import ensure_numeric if file_text[-4:] != '.txt': raise IOError('Input file %s should be of type .txt.' % file_text) if file_out is None: file_out = file_text[:-4] + '.tms' fid = open(file_text) line = fid.readline() fid.close() fields = line.split(',') msg = "File %s must have the format 'datetime, value0 value1 value2 ...'" \ % file_text assert len(fields) == 2, msg if not time_as_seconds: try: starttime = calendar.timegm(time.strptime(fields[0], time_format)) except ValueError: msg = 'First field in file %s must be' % file_text msg += ' date-time with format %s.\n' % time_format msg += 'I got %s instead.' % fields[0] raise DataTimeError, msg else: try: starttime = float(fields[0]) except Error: msg = "Bad time format" raise DataTimeError, msg # Split values values = [] for value in fields[1].split(): values.append(float(value)) q = ensure_numeric(values) msg = 'ERROR: File must contain at least one independent value' assert len(q.shape) == 1, msg # Read times proper from anuga.config import time_format import time, calendar fid = open(file_text) lines = fid.readlines() fid.close() N = len(lines) d = len(q) T = num.zeros(N, num.float) # Time Q = num.zeros((N, d), num.float) # Values for i, line in enumerate(lines): fields = line.split(',') if not time_as_seconds: realtime = calendar.timegm(time.strptime(fields[0], time_format)) else: realtime = float(fields[0]) T[i] = realtime - starttime for j, value in enumerate(fields[1].split()): Q[i, j] = float(value) msg = 'File %s must list time as a monotonuosly ' % file_text msg += 'increasing sequence' assert num.alltrue(T[1:] - T[:-1] > 0), msg #Create NetCDF file fid = NetCDFFile(file_out, netcdf_mode_w) fid.institution = 'Geoscience Australia' fid.description = 'Time series' #Reference point #Start time in seconds since the epoch (midnight 1/1/1970) #FIXME: Use Georef fid.starttime = starttime # dimension definitions #fid.createDimension('number_of_volumes', self.number_of_volumes) #fid.createDimension('number_of_vertices', 3) fid.createDimension('number_of_timesteps', len(T)) fid.createVariable('time', netcdf_float, ('number_of_timesteps',)) fid.variables['time'][:] = T for i in range(Q.shape[1]): try: name = quantity_names[i] except: name = 'Attribute%d' % i fid.createVariable(name, netcdf_float, ('number_of_timesteps',)) fid.variables[name][:] = Q[:,i] fid.close()
def _convert_dem_from_ascii2netcdf(name_in, name_out = None, verbose = False): """Read Digital Elevation model from the following ASCII format (.asc) Internal function. See public function convert_dem_from_ascii2netcdf for details. """ import os from anuga.file.netcdf import NetCDFFile root = name_in[:-4] # Read Meta data if verbose: log.critical('Reading METADATA from %s' % (root + '.prj')) metadatafile = open(root + '.prj') metalines = metadatafile.readlines() metadatafile.close() L = metalines[0].strip().split() assert L[0].strip().lower() == 'projection' projection = L[1].strip() #TEXT L = metalines[1].strip().split() assert L[0].strip().lower() == 'zone' zone = int(L[1].strip()) L = metalines[2].strip().split() assert L[0].strip().lower() == 'datum' datum = L[1].strip() #TEXT L = metalines[3].strip().split() assert L[0].strip().lower() == 'zunits' #IGNORE zunits = L[1].strip() #TEXT L = metalines[4].strip().split() assert L[0].strip().lower() == 'units' units = L[1].strip() #TEXT L = metalines[5].strip().split() assert L[0].strip().lower() == 'spheroid' #IGNORE spheroid = L[1].strip() #TEXT L = metalines[6].strip().split() assert L[0].strip().lower() == 'xshift' false_easting = float(L[1].strip()) L = metalines[7].strip().split() assert L[0].strip().lower() == 'yshift' false_northing = float(L[1].strip()) if name_in[-4:] != '.asc': raise IOError('Input file %s should be of type .asc.' % name_in) #Read DEM data datafile = open(name_in) if verbose: log.critical('Reading DEM from %s' % (name_in)) lines = datafile.readlines() datafile.close() if verbose: log.critical('Got %d lines' % len(lines)) ncols = int(lines[0].split()[1].strip()) nrows = int(lines[1].split()[1].strip()) # Do cellsize (line 4) before line 2 and 3 cellsize = float(lines[4].split()[1].strip()) # Checks suggested by Joaquim Luis # Our internal representation of xllcorner # and yllcorner is non-standard. xref = lines[2].split() if xref[0].strip() == 'xllcorner': xllcorner = float(xref[1].strip()) # + 0.5*cellsize # Correct offset elif xref[0].strip() == 'xllcenter': xllcorner = float(xref[1].strip()) else: msg = 'Unknown keyword: %s' % xref[0].strip() raise Exception, msg yref = lines[3].split() if yref[0].strip() == 'yllcorner': yllcorner = float(yref[1].strip()) # + 0.5*cellsize # Correct offset elif yref[0].strip() == 'yllcenter': yllcorner = float(yref[1].strip()) else: msg = 'Unknown keyword: %s' % yref[0].strip() raise Exception, msg NODATA_value = int(float(lines[5].split()[1].strip())) assert len(lines) == nrows + 6 if name_out == None: netcdfname = name_in[:-4]+'.dem' else: netcdfname = name_out + '.dem' if verbose: log.critical('Store to NetCDF file %s' % netcdfname) # NetCDF file definition fid = NetCDFFile(netcdfname, netcdf_mode_w) #Create new file fid.institution = 'Geoscience Australia' fid.description = 'NetCDF DEM format for compact and portable storage ' \ 'of spatial point data' fid.ncols = ncols fid.nrows = nrows fid.xllcorner = xllcorner fid.yllcorner = yllcorner fid.cellsize = cellsize fid.NODATA_value = NODATA_value fid.zone = zone fid.false_easting = false_easting fid.false_northing = false_northing fid.projection = projection fid.datum = datum fid.units = units # dimension definitions fid.createDimension('number_of_rows', nrows) fid.createDimension('number_of_columns', ncols) # variable definitions fid.createVariable('elevation', netcdf_float, ('number_of_rows', 'number_of_columns')) # Get handles to the variables elevation = fid.variables['elevation'] #Store data import numpy datafile = open(name_in) elevation[:,:] = numpy.loadtxt(datafile, skiprows=6) datafile.close() # n = len(lines[6:]) # for i, line in enumerate(lines[6:]): # fields = line.split() # if verbose and i % ((n+10)/10) == 0: # log.critical('Processing row %d of %d' % (i, nrows)) # # if len(fields) != ncols: # msg = 'Wrong number of columns in file "%s" line %d\n' % (name_in, i) # msg += 'I got %d elements, but there should have been %d\n' % (len(fields), ncols) # raise Exception, msg # # elevation[i, :] = num.array([float(x) for x in fields]) fid.close()
def _generic_dem2pts( self, name_in, name_out=None, quantity_name=None, verbose=False, easting_min=None, easting_max=None, northing_min=None, northing_max=None, ): """Read raster from the following NetCDF format (.dem) Internal function. See public function generic_dem2pts for details. """ # FIXME: Can this be written feasibly using write_pts? import os from anuga.file.netcdf import NetCDFFile root = name_in[:-4] if name_in[-4:] == ".asc": intermediate = root + ".dem" if verbose: log.critical("Preconvert %s from asc to %s" % (name_in, intermediate)) asc2dem(name_in) name_in = intermediate elif name_in[-4:] != ".dem": raise IOError("Input file %s should be of type .asc or .dem." % name_in) if name_out != None and basename_out[-4:] != ".pts": raise IOError("Input file %s should be of type .pts." % name_out) # Get NetCDF infile = NetCDFFile(name_in, netcdf_mode_r) if verbose: log.critical("Reading raster from %s" % (name_in)) ncols = int(infile.ncols) nrows = int(infile.nrows) xllcorner = float(infile.xllcorner) # Easting of lower left corner yllcorner = float(infile.yllcorner) # Northing of lower left corner cellsize = float(infile.cellsize) NODATA_value = float(infile.NODATA_value) dem_elevation = infile.variables[quantity_name] zone = int(infile.zone) false_easting = float(infile.false_easting) false_northing = float(infile.false_northing) # print ncols, nrows, xllcorner,yllcorner, cellsize, NODATA_value, zone # Text strings projection = infile.projection datum = infile.datum units = infile.units # print projection, datum, units # Get output file if name_out == None: ptsname = root + ".pts" else: ptsname = name_out if verbose: log.critical("Store to NetCDF file %s" % ptsname) # NetCDF file definition outfile = NetCDFFile(ptsname, netcdf_mode_w) # Create new file outfile.institution = "Geoscience Australia" outfile.description = "NetCDF pts format for compact and portable " "storage of spatial point data" # Assign default values if easting_min is None: easting_min = xllcorner if easting_max is None: easting_max = xllcorner + ncols * cellsize if northing_min is None: northing_min = yllcorner if northing_max is None: northing_max = yllcorner + nrows * cellsize # print easting_min, easting_max, northing_min, northing_max # Compute offsets to update georeferencing easting_offset = xllcorner - easting_min northing_offset = yllcorner - northing_min # Georeferencing outfile.zone = zone outfile.xllcorner = easting_min # Easting of lower left corner outfile.yllcorner = northing_min # Northing of lower left corner outfile.false_easting = false_easting outfile.false_northing = false_northing outfile.projection = projection outfile.datum = datum outfile.units = units # Grid info (FIXME: probably not going to be used, but heck) outfile.ncols = ncols outfile.nrows = nrows dem_elevation_r = num.reshape(dem_elevation, (nrows, ncols)) totalnopoints = nrows * ncols # ======================================== # Do the preceeding with numpy # ======================================== y = num.arange(nrows, dtype=num.float) y = yllcorner + (nrows - 1) * cellsize - y * cellsize x = num.arange(ncols, dtype=num.float) x = xllcorner + x * cellsize xx, yy = num.meshgrid(x, y) xx = xx.flatten() yy = yy.flatten() flag = num.logical_and( num.logical_and((xx <= easting_max), (xx >= easting_min)), num.logical_and((yy <= northing_max), (yy >= northing_min)), ) dem = dem_elevation[:].flatten() id = num.where(flag)[0] xx = xx[id] yy = yy[id] dem = dem[id] clippednopoints = len(dem) # print clippedpoints # print xx # print yy # print dem data_flag = dem != NODATA_value data_id = num.where(data_flag) xx = xx[data_id] yy = yy[data_id] dem = dem[data_id] nn = clippednopoints - len(dem) nopoints = len(dem) if verbose: log.critical("There are %d values in the raster" % totalnopoints) log.critical("There are %d values in the clipped raster" % clippednopoints) log.critical("There are %d NODATA_values in the clipped raster" % nn) outfile.createDimension("number_of_points", nopoints) outfile.createDimension("number_of_dimensions", 2) # This is 2d data # Variable definitions outfile.createVariable("points", netcdf_float, ("number_of_points", "number_of_dimensions")) outfile.createVariable(quantity_name, netcdf_float, ("number_of_points",)) # Get handles to the variables points = outfile.variables["points"] elevation = outfile.variables[quantity_name] points[:, 0] = xx - easting_min points[:, 1] = yy - northing_min elevation[:] = dem infile.close() outfile.close()
def _write_msh_file(file_name, mesh): """Write .msh NetCDF file WARNING: This function mangles the mesh data structure """ # FIXME(Ole and John): We ran into a problem on Bogong (64 bit) # where integers appeared as arrays. This may be similar to # problem seen by Steve in changeset:2778 where he had to wrap # them in int. Now we are trying with the native Integer format # (Int == 'l' == Int64). However, that caused casting errors, when # 64bit arrays are to be assigned to their NetCDF counterparts. It # seems that the NetCDF arrays are 32bit even though they are # created with the type Int64. Need to look at the NetCDF library # in more detail. IntType = num.int32 #IntType = Int #print 'mesh vertices',mesh['vertices'].shape #the triangulation mesh['vertices'] = num.array(mesh['vertices'], num.float) mesh['vertex_attribute_titles'] = \ num.array(string_to_char(mesh['vertex_attribute_titles']), num.character) num_attributes = len(mesh['vertex_attribute_titles']) num_vertices = mesh['vertices'].shape[0] #print 'num_attrib ',num_attributes if mesh['vertex_attributes'] != None: mesh['vertex_attributes'] = \ num.array(mesh['vertex_attributes'], num.float) if num_attributes > 0: mesh['vertex_attributes'] = \ num.reshape(mesh['vertex_attributes'],(num_vertices,-1)) mesh['segments'] = num.array(mesh['segments'], IntType) mesh['segment_tags'] = num.array(string_to_char(mesh['segment_tags']), num.character) mesh['triangles'] = num.array(mesh['triangles'], IntType) mesh['triangle_tags'] = num.array(string_to_char(mesh['triangle_tags']), num.character) mesh['triangle_neighbors'] = \ num.array(mesh['triangle_neighbors'], IntType) #the outline mesh['points'] = num.array(mesh['points'], num.float) mesh['point_attributes'] = num.array(mesh['point_attributes'], num.float) mesh['outline_segments'] = num.array(mesh['outline_segments'], IntType) mesh['outline_segment_tags'] = \ num.array(string_to_char(mesh['outline_segment_tags']), num.character) mesh['holes'] = num.array(mesh['holes'], num.float) mesh['regions'] = num.array(mesh['regions'], num.float) mesh['region_tags'] = num.array(string_to_char(mesh['region_tags']), num.character) mesh['region_max_areas'] = num.array(mesh['region_max_areas'], num.float) # NetCDF file definition try: outfile = NetCDFFile(file_name, netcdf_mode_w) except IOError: msg = 'File %s could not be created' % file_name raise Exception, msg #Create new file outfile.institution = 'Geoscience Australia' outfile.description = 'NetCDF format for compact and portable storage ' + \ 'of spatial point data' # dimension definitions - fixed outfile.createDimension('num_of_dimensions', 2) # This is 2d data outfile.createDimension('num_of_segment_ends', 2) # Segs have two points outfile.createDimension('num_of_triangle_vertices', 3) outfile.createDimension('num_of_triangle_faces', 3) outfile.createDimension('num_of_region_max_area', 1) # Create dimensions, variables and set the variables # trianglulation # vertices if (mesh['vertices'].shape[0] > 0): outfile.createDimension('num_of_vertices', mesh['vertices'].shape[0]) outfile.createVariable('vertices', netcdf_float, ('num_of_vertices', 'num_of_dimensions')) outfile.variables['vertices'][:] = mesh['vertices'] #print 'mesh vertex attributes', mesh['vertex_attributes'].shape if (mesh['vertex_attributes'] is not None and (mesh['vertex_attributes'].shape[0] > 0 and mesh['vertex_attributes'].shape[1] > 0)): outfile.createDimension('num_of_vertex_attributes', mesh['vertex_attributes'].shape[1]) outfile.createDimension('num_of_vertex_attribute_title_chars', mesh['vertex_attribute_titles'].shape[1]) outfile.createVariable( 'vertex_attributes', netcdf_float, ('num_of_vertices', 'num_of_vertex_attributes')) outfile.createVariable('vertex_attribute_titles', netcdf_char, ('num_of_vertex_attributes', 'num_of_vertex_attribute_title_chars')) outfile.variables['vertex_attributes'][:] = \ mesh['vertex_attributes'] outfile.variables['vertex_attribute_titles'][:] = \ mesh['vertex_attribute_titles'] # segments if (mesh['segments'].shape[0] > 0): outfile.createDimension('num_of_segments', mesh['segments'].shape[0]) outfile.createVariable('segments', netcdf_int, ('num_of_segments', 'num_of_segment_ends')) outfile.variables['segments'][:] = mesh['segments'] if (mesh['segment_tags'].shape[1] > 0): outfile.createDimension('num_of_segment_tag_chars', mesh['segment_tags'].shape[1]) outfile.createVariable( 'segment_tags', netcdf_char, ('num_of_segments', 'num_of_segment_tag_chars')) outfile.variables['segment_tags'][:] = mesh['segment_tags'] # triangles if (mesh['triangles'].shape[0] > 0): outfile.createDimension('num_of_triangles', mesh['triangles'].shape[0]) outfile.createVariable( 'triangles', netcdf_int, ('num_of_triangles', 'num_of_triangle_vertices')) outfile.createVariable('triangle_neighbors', netcdf_int, ('num_of_triangles', 'num_of_triangle_faces')) outfile.variables['triangles'][:] = mesh['triangles'] outfile.variables['triangle_neighbors'][:] = mesh['triangle_neighbors'] if (mesh['triangle_tags'] is not None and (mesh['triangle_tags'].shape[1] > 0)): outfile.createDimension('num_of_triangle_tag_chars', mesh['triangle_tags'].shape[1]) outfile.createVariable( 'triangle_tags', netcdf_char, ('num_of_triangles', 'num_of_triangle_tag_chars')) outfile.variables['triangle_tags'][:] = mesh['triangle_tags'] # outline # points if (mesh['points'].shape[0] > 0): outfile.createDimension('num_of_points', mesh['points'].shape[0]) outfile.createVariable('points', netcdf_float, ('num_of_points', 'num_of_dimensions')) outfile.variables['points'][:] = mesh['points'] if mesh['point_attributes'].shape[0] > 0 \ and mesh['point_attributes'].shape[1] > 0: outfile.createDimension('num_of_point_attributes', mesh['point_attributes'].shape[1]) outfile.createVariable( 'point_attributes', netcdf_float, ('num_of_points', 'num_of_point_attributes')) outfile.variables['point_attributes'][:] = mesh['point_attributes'] # outline_segments if mesh['outline_segments'].shape[0] > 0: outfile.createDimension('num_of_outline_segments', mesh['outline_segments'].shape[0]) outfile.createVariable( 'outline_segments', netcdf_int, ('num_of_outline_segments', 'num_of_segment_ends')) outfile.variables['outline_segments'][:] = mesh['outline_segments'] if mesh['outline_segment_tags'].shape[1] > 0: outfile.createDimension('num_of_outline_segment_tag_chars', mesh['outline_segment_tags'].shape[1]) outfile.createVariable('outline_segment_tags', netcdf_char, ('num_of_outline_segments', 'num_of_outline_segment_tag_chars')) outfile.variables['outline_segment_tags'][:] = \ mesh['outline_segment_tags'] # holes if (mesh['holes'].shape[0] > 0): outfile.createDimension('num_of_holes', mesh['holes'].shape[0]) outfile.createVariable('holes', netcdf_float, ('num_of_holes', 'num_of_dimensions')) outfile.variables['holes'][:] = mesh['holes'] # regions if (mesh['regions'].shape[0] > 0): outfile.createDimension('num_of_regions', mesh['regions'].shape[0]) outfile.createVariable('regions', netcdf_float, ('num_of_regions', 'num_of_dimensions')) outfile.createVariable('region_max_areas', netcdf_float, ('num_of_regions', )) outfile.variables['regions'][:] = mesh['regions'] outfile.variables['region_max_areas'][:] = mesh['region_max_areas'] if (mesh['region_tags'].shape[1] > 0): outfile.createDimension('num_of_region_tag_chars', mesh['region_tags'].shape[1]) outfile.createVariable( 'region_tags', netcdf_char, ('num_of_regions', 'num_of_region_tag_chars')) outfile.variables['region_tags'][:] = mesh['region_tags'] # geo_reference info if mesh.has_key('geo_reference') and not mesh['geo_reference'] is None: mesh['geo_reference'].write_NetCDF(outfile) outfile.close()
def test_decimate_dem(self): """Test decimation of dem file """ import os from anuga.file.netcdf import NetCDFFile # Write test dem file root = "decdemtest" filename = root + ".dem" fid = NetCDFFile(filename, netcdf_mode_w) fid.institution = "Geoscience Australia" fid.description = "NetCDF DEM format for compact and portable " + "storage of spatial point data" nrows = 15 ncols = 18 fid.ncols = ncols fid.nrows = nrows fid.xllcorner = 2000.5 fid.yllcorner = 3000.5 fid.cellsize = 25 fid.NODATA_value = -9999 fid.zone = 56 fid.false_easting = 0.0 fid.false_northing = 0.0 fid.projection = "UTM" fid.datum = "WGS84" fid.units = "METERS" fid.createDimension("number_of_points", nrows * ncols) fid.createVariable("elevation", netcdf_float, ("number_of_points",)) elevation = fid.variables["elevation"] elevation[:] = num.arange(nrows * ncols) fid.close() # generate the elevation values expected in the decimated file ref_elevation = [ (0 + 1 + 2 + 18 + 19 + 20 + 36 + 37 + 38) / 9.0, (4 + 5 + 6 + 22 + 23 + 24 + 40 + 41 + 42) / 9.0, (8 + 9 + 10 + 26 + 27 + 28 + 44 + 45 + 46) / 9.0, (12 + 13 + 14 + 30 + 31 + 32 + 48 + 49 + 50) / 9.0, (72 + 73 + 74 + 90 + 91 + 92 + 108 + 109 + 110) / 9.0, (76 + 77 + 78 + 94 + 95 + 96 + 112 + 113 + 114) / 9.0, (80 + 81 + 82 + 98 + 99 + 100 + 116 + 117 + 118) / 9.0, (84 + 85 + 86 + 102 + 103 + 104 + 120 + 121 + 122) / 9.0, (144 + 145 + 146 + 162 + 163 + 164 + 180 + 181 + 182) / 9.0, (148 + 149 + 150 + 166 + 167 + 168 + 184 + 185 + 186) / 9.0, (152 + 153 + 154 + 170 + 171 + 172 + 188 + 189 + 190) / 9.0, (156 + 157 + 158 + 174 + 175 + 176 + 192 + 193 + 194) / 9.0, (216 + 217 + 218 + 234 + 235 + 236 + 252 + 253 + 254) / 9.0, (220 + 221 + 222 + 238 + 239 + 240 + 256 + 257 + 258) / 9.0, (224 + 225 + 226 + 242 + 243 + 244 + 260 + 261 + 262) / 9.0, (228 + 229 + 230 + 246 + 247 + 248 + 264 + 265 + 266) / 9.0, ] # generate a stencil for computing the decimated values stencil = num.ones((3, 3), num.float) / 9.0 dem2dem(filename, stencil=stencil, cellsize_new=100) # Open decimated NetCDF file fid = NetCDFFile(root + "_100.dem", netcdf_mode_r) # Get decimated elevation elevation = fid.variables["elevation"] # Check values assert num.allclose(elevation, ref_elevation) # Cleanup fid.close() os.remove(root + ".dem") os.remove(root + "_100.dem")