def define_bdy(coordsfile, meshfile, grid): # this function initialises the BDY and store all important info in a dictionnary BDY = {} # gather the initial cell centre depth of the domain grid points fmesh = DS(meshfile) depth_0 = fmesh.variables['gdept'][:].squeeze() depth_levels = depth_0.shape[0] cell_thickness = fmesh.variables['e3' + grid.lower()][:].squeeze() fmesh.close() # extract the lat, lon coordinates of the boundary from the bdy_coordinate file # bdy_i, bdy_j are the grid indices of the boundary points # bdy_lat, bdy_lon are the coorindates of the boundary points # bdy_r is the "r" index in the bdy, i.e. the ordinal number of the bdy layer (generally the baoundary is imposed on a band along the bpoundary that is R cells thick fcoords = DS(coordsfile) BDY['nbj'] = fcoords.variables['nbj' + grid.lower()][:].squeeze() - 1 BDY['nbi'] = fcoords.variables['nbi' + grid.lower()][:].squeeze() - 1 BDY['lat'] = fcoords.variables['gphi' + grid.lower()][:].squeeze() BDY['lon'] = fcoords.variables['glam' + grid.lower()][:].squeeze() BDY['nbr'] = fcoords.variables['nbr' + grid.lower()][:].squeeze() fcoords.close() #extract depth for all bdy points: #bdy_depth=zeros((depth_levels,BDY['nbj'].size)) #bdy_cell_thickness=zeros((depth_levels,BDY['nbj'].size)) #for position in range(BDY['nbj'].size): # bdy_depth[:,position]=depth_0[:,BDY['nbj'][position],BDY['nbi'][position]] # bdy_cell_thickness[:,position]=cell_thickness[:,BDY['nbj'][position],BDY['nbi'][position]] BDY['depth'] = depth_0.data #bdy_depth BDY['e3'] = cell_thickness.data #bdy_cell_thickness return BDY
def write_bdy(var_to_write, ncname, varname, y0=1960, y1=2099, climatology=False): # this write the BDY in a NEMO netcdf BDY file count = -1 if climatology == False and var_to_write.shape[0] != (y1 - y0 + 1) * 12: print( 'time size of the array does not correspondes with the dates provided' ) print('size: ', var_to_write.shape[0], 'dates: %4i-%4i' % (y0, y1)) return if len(var_to_write.shape) == 3: #save 3D variable for y in range(y0, y1 + 1): if mod(y, 10) == 0: print('saving year %3i' % y) for m in range(1, 13): count = m - 1 if climatology else count + 1 ncfile = DS(ncname + '_y%4im%02i.nc' % (y, m), 'a') ncfile.variables[varname][0, :, 0, :] = var_to_write[count] ncfile.close() elif len(var_to_write.shape) == 2: # save 2D variables like barotropic velocities and ssh for y in range(y0, y1 + 1): if mod(y, 10) == 0: print('saving year %3i' % y) for m in range(1, 13): ncfile = DS(ncname + '_y%4im%02i.nc' % (y, m), 'a') ncfile.variables[varname][0, 0, :] = var_to_write[count] count += 1 ncfile.close() return
def extract_values(BDY, varfile, varname, scaling=1., bias=0., fill_mask=False): # this function extracte the variable "varname" from the "varfile" netcdf file uding the BDY metadata in the BDY dictionnary # if the surface monthly values need to be extracted then the logical switch needs to be set to true and the name of the netcdf file with the surface monthly value need to be read. # it also extract the original depth information # open the files and check what is the name of the depth variable fvar = DS(varfile) if 'lev' in fvar.variables.keys(): zvar = 'lev' if 'nav_lev' in fvar.variables.keys(): zvar = 'nav_lev' zdim = 'z' elif 'depth' in fvar.variables.keys(): zvar = 'depth' zdim = 'depth' else: print('No recognised depth array in the file') zvar = '' #extract variable # if the length of the time dimension is big (i.e. monthly), then extract data one timstep at time to save memory print('extracting values for ' + varname) timelen = len(fvar.dimensions['t']) if zvar != '': nlev = len(fvar.dimensions[zdim]) var = zeros((timelen, nlev, BDY['nbj'].size)) else: var = zeros((timelen, BDY['nbj'].size)) print('given the big size the extraction is one timestep at time') print('this might take few minutes') for time in range(timelen): tmp_var = fvar.variables[varname][time] if zvar != '': var[time] = tmp_var[:, BDY['nbj'], BDY['nbi']] * scaling - bias else: var[time] = tmp_var[BDY['nbj'], BDY['nbi']] * scaling - bias if mod(time, 50) == 0: print('%4i out of %4i' % (time, timelen)) # extract depth information if zvar != '': original_depth = fvar.variables[zvar][:] else: original_depth = 0. fvar.close() return var, original_depth
import numpy as np import matplotlib.pyplot as plt from netCDF4 import Dataset as DS f = DS("../scmdiag-output.nc","r") time = f.dimensions['time'] ntime = len(time) plt.figure( figsize=(10,6) ) cp = 1004 lat = 2.5e6 fc = 3600*24 camttend_all = fc/cp*f.variables['camstend'][:,::-1,0].T camttend = np.mean(fc/cp*f.variables['camstend'][:,::-1,0], axis=0) camttendcond = np.mean(fc/cp*f.variables['camstendcond'][:,::-1,0], axis=0) camttendtranup = np.mean(fc/cp*f.variables['camstendtranup'][:,::-1,0], axis=0) camttendtrandn = np.mean(fc/cp*f.variables['camstendtrandn'][:,::-1,0], axis=0) camqtend_all = lat/cp*fc*f.variables['camqtend'][:,::-1,0].T camqtend = np.mean(lat/cp*fc*f.variables['camqtend'][:,::-1,0], axis=0) camqtendcond = np.mean(lat/cp*fc*f.variables['camqtendcond'][:,::-1,0], axis=0) camqtendtranup = np.mean(lat/cp*fc*f.variables['camqtendtranup'][:,::-1,0], axis=0) camqtendtrandn = np.mean(lat/cp*fc*f.variables['camqtendtrandn'][:,::-1,0], axis=0) qcheck = f.variables['qcheck'][:,0]
def create_file(filename, config, coords, gridchar): #this function create an empty boundary file to be used in the extract_BDY script # it requires as input: # filename: filename to create # config: the dictionnary containing all configuration options associated to that file from the yaml file # coords: a dictionnary containing the coordinates of the bdy points both in i,j,r and lon,lat nr = coords['nbr' + gridchar].max() dims = coords['nbi' + gridchar].shape print('creating ' + filename) ncfile = DS(filename, 'w') ncfile.createDimension('time_counter', None) timedim = 1 ncfile.createDimension('xb', dims[1]) ncfile.createDimension('yb', dims[0]) var_layers = [] for varname in coords.keys(): #create and copy coordinates variables if varname[0] == 'n': ncfile.createVariable(varname, 'int', ('yb', 'xb')) ncfile.variables[varname][:] = coords[varname] if varname[0] == 'g': ncfile.createVariable(varname, 'f4', ('yb', 'xb')) ncfile.variables[varname][:] = coords[varname] for varname in config['variables'].keys(): var_layers += [ config['variables'][varname]['depth'], ] max_depth = array(var_layers).max() check = (var_layers == max_depth) | (var_layers == zeros(len(var_layers))) if not (check.all()): print("variables in %s have different numbers of z layers" % filename) print("be sure they are either 0 or the same number") sys.exit() if max_depth > 0: ncfile.createDimension('zb', max_depth) ncfile.creation_date = datetime.now().isoformat() ncfile.rim_width = '%iLL' % nr for at in config.keys(): if at == 'variables': continue ncfile.setncattr(at, config[at]) var_config = config['variables'] for varname in var_config.keys(): if var_config[varname]['depth'] > 0: ncfile.createVariable(varname, 'f4', ('time_counter', 'zb', 'yb', 'xb'), zlib=True, complevel=9, fill_value=FV['f4']) ncfile.variables[varname][:] = zeros( (timedim, max_depth, dims[0], dims[1])) + FV['f4'] else: ncfile.createVariable(varname, 'f4', ('time_counter', 'yb', 'xb'), zlib=True, complevel=9, fill_value=FV['f4']) ncfile.variables[varname][:] = zeros( (timedim, dims[0], dims[1])) + FV['f4'] ncfile.variables[varname].grid = 'bdy' + gridchar.upper() for at in var_config[varname].keys(): if at == 'depth': continue ncfile.variables[varname].setncattr(at, var_config[varname][at]) ncfile.close() return
monthly = True try: daily = Yconfiguration['daily'] except: daily = False if daily & monthly: print('The script has been configured to be both daily and monthly') print( 'this is not currently allowed, if you need to generate both types, create two configuration files and run the script twice' ) print('now only the MONTHLY files will be generated') daily = False for Yfile in Yconfiguration['files'].keys(): fcoords = DS(coords_file) grid = Yconfiguration['files'][Yfile]['grid'] gridchar = grid.lower() coords = {} coords['nbi' + gridchar] = fcoords.variables['nbi' + gridchar][:] coords['nbj' + gridchar] = fcoords.variables['nbj' + gridchar][:] coords['nbr' + gridchar] = fcoords.variables['nbr' + gridchar][:] coords['gphi' + gridchar] = fcoords.variables['gphi' + gridchar][:] coords['glam' + gridchar] = fcoords.variables['glam' + gridchar][:] try: rim = min(coords['nbr' + gridchar].max(), Yconfiguration['files'][Yfile]['rim']) except: rim = coords['nbr' + gridchar].max() if rim != coords['nbr' + gridchar].max(): dims = coords['nbi' + gridchar].shape
def read_netcdf(netcdf_filename, varname, outputdir, outfile, year): ''' Read out of data "varname" out of netcdf file. If year==None, then all years are written to outputdirectory. ''' # Location and name specification of netcdf files. #pcglobwb_dir = "PCR_Routing/netCDF" #basename_netcdf = "pcrglobwb_CRU21_30min_" #extension_netcdf = "_yearly.nc" #outputdir = "pcr_routing_out" # Get the command line arguments #year = sys.argv[1] #varname = sys.argv[2] #outfile = sys.argv[3] #extensie =sys.argv[4] #extension_netcdf = extensie + extension_netcdf if (not os.path.isdir(outputdir)): os.mkdir(outputdir) #netcdf_file = os.path.join(pcglobwb_dir,basename_netcdf + varname + extension_netcdf) output_file = os.path.join(os.path.join(outputdir, str(year)), outfile) if (not os.path.isfile(netcdf_filename)): print("***** ERROR *****") print("File is not found.") print(("Looked for file: " + netcdf_filename)) print("***** ERROR *****") raise IOError("File " + netcdf_filename + " is not found") # Load netcdf file nc = DS(netcdf_filename) # Check whether varname is in the netcdf file if (not varname in list(nc.variables.keys())): # There is a problem, varname is not available. print((varname + " is not found in file: " + netcdf_filename)) print(("Parameters available: " + ",".join(map(str, list(nc.variables.keys()))))) raise IOError("Parameter " + varname + " is not found in file " + netcdf_filename) # Get the time settings of the nc file try: cdftime = utime(nc.variables['time'].units, nc.variables['time'].calendar) start_year = cdftime.num2date(nc.variables['time'][0]).year end_year = cdftime.num2date(nc.variables['time'][-1]).year except AttributeError: # File is not time dependent. # Close netcdf file nc.close() read_netcdf_constant(netcdf_filename, varname, outputdir, outfile) if (year < start_year or year > end_year): print("***** ERROR *****") print(("Year: " + str(year) + " is not found in netcdf file " + netcdf_filename)) print(("Years can be given between " + str(start_year) + " and " + str(end_year))) print("***** ERROR *****") raise IOError("Year " + str(year) + " is outside the year range in file " + netcdf_filename) else: # Find the year in the netcdf file for i in range(len(nc.variables['time'])): if (cdftime.num2date(nc.variables['time'][i]).year == int(year)): iyear = i break else: print("***** ERROR *****") print(("Specified year " + str(year) + " is not found in netcdf file " + netcdf_filename)) print("Years available: ") for i in range(len(cdftime.num2date(nc.variables['time']))): print(cdftime.num2date(nc.variables['time'][i]).year) raise IOError("Year " + str(int(year)) + " is not found in file " + netcdf_filename) # Get the variable. nc_var = nc.variables[varname][iyear, :, :] lat = nc.variables["latitude"][:] lon = nc.variables["longitude"][:] nrows = len(lat) ncols = len(lon) # Close netcdf file nc.close() dim = nc_var.shape if (dim[0] != nrows or dim[1] != ncols): print(("File: " + netcdf_filename + " is not consistent.")) print(("In parameter file: nrows = ", nrows, " and ncols = ", ncols)) print(("In netcdf file: nrows = ", dim[0], " and ncols = ", dim[1])) # General settings for ascii grid format ncols = 720 nrows = 360 xll = -180 yll = -90 cellsize = 0.5 nodata_value = -9999 # Create a raster without a mask and filled with nodata. out_grid = ascraster.Asciigrid(ncols=ncols,nrows=nrows,\ xllcorner=xll, \ yllcorner = yll, \ cellsize = cellsize,\ nodata_value = nodata_value,\ numtype=float) # Put the nc_var data into an ascraster object for i in range(nrows): for j in range(ncols): # Check whether there is no data if (type(nc_var[i, j]) != numpy.ma.core.MaskedConstant): icell = out_grid.get_index_from_coordin(lon[j], lat[i]) out_grid.set_data(icell, nc_var[i, j]) if (not os.path.isdir(os.path.join(outputdir, year))): os.mkdir(os.path.join(outputdir, year)) out_grid.write_ascii_file(output_file)
def read_netcdf_constant(netcdf_filename, varname, outputdir, outfile): ''' Read out of data "varname" out of netcdf file. If year==None, then all years are written to outputdirectory. ''' if (not os.path.isdir(outputdir)): os.mkdir(outputdir) # Make output file output_file = os.path.join(outputdir, outfile) if (not os.path.isfile(netcdf_filename)): print("***** ERROR *****") print("File is not found.") print(("Looked for file: " + netcdf_filename)) print("***** ERROR *****") raise IOError("File " + netcdf_filename + " is not found") # Load netcdf file nc = DS(netcdf_filename) # Check whether varname is in the netcdf file if (not varname in list(nc.variables.keys())): # There is a problem, varname is not available. print((varname + " is not found in file: " + netcdf_filename)) print(("Parameters available: " + ",".join(map(str, list(nc.variables.keys()))))) raise IOError("Parameter " + varname + " is not found in file " + netcdf_filename) # Get the variable. nc_var = nc.variables[varname][:, :] try: lat = nc.variables["latitude"][:] except KeyError: lat = nc.variables["lat"][:] try: lon = nc.variables["longitude"][:] except KeyError: lon = nc.variables["lon"][:] nrows = len(lat) ncols = len(lon) # Close netcdf file nc.close() dim = nc_var.shape if (dim[0] != nrows or dim[1] != ncols): print(("File: " + netcdf_filename + " is not consistent.")) print(("In parameter file: nrows = ", nrows, " and ncols = ", ncols)) print(("In netcdf file: nrows = ", dim[0], " and ncols = ", dim[1])) # General settings for ascii grid format ncols = 720 nrows = 360 xll = -180 yll = -90 cellsize = 0.5 nodata_value = -9999 # Create a raster without a mask and filled with nodata. out_grid = ascraster.Asciigrid(ncols=ncols,nrows=nrows,\ xllcorner=xll, \ yllcorner = yll, \ cellsize = cellsize,\ nodata_value = nodata_value,\ numtype=float) # Put the nc_var data into an ascraster object for i in range(nrows): for j in range(ncols): # Check whether there is no data if (type(nc_var[i, j]) != numpy.ma.core.MaskedConstant): icell = out_grid.get_index_from_coordin(lon[j], lat[i]) out_grid.set_data(icell, nc_var[i, j]) if (not os.path.isdir(os.path.join(outputdir, year))): os.mkdir(os.path.join(outputdir, year)) out_grid.write_ascii_file(output_file)
} #here the information about themetadata are read and sotred tin the BDY dictionnary BDY = define_bdy(coordsfile, meshfile, grid) for varname in variables.keys(): var_keys = Yconfiguration['variables'][varname] print('Processing %s' % varname) if 'fixed_value' in var_keys.keys(): outfile = var_keys['output_file'] outvar = out_dict[varname] value = var_keys['fixed_value'] print('Applying fixed value of %s' % str(value)) tmp_file = DS(outfile + '_y%4im%02i.nc' % (ystart, 1)) tmp_dims = tmp_file.variables[outvar][:].shape tmp_file.close() output_val = value * ones( ((yend - ystart + 1) * 12, tmp_dims[1], tmp_dims[3])) write_bdy(output_val, outfile, outvar, y0=ystart, y1=yend) continue elif 'input_file' in var_keys.keys(): # extract the BGC variable from the reference dataset print('Extracting field from %s' % var_keys['input_file']) output_val, input_depth = extract_values(BDY, var_keys['input_file'], infile_dict[varname]) climatology = True if 'climatology' in var_keys else False