def define_bdy(coordsfile, meshfile, grid):
# this function initialises the BDY and store all important info in a dictionnary
BDY = {}
# gather the initial cell centre depth of the domain grid points
fmesh = DS(meshfile)
depth_0 = fmesh.variables['gdept'][:].squeeze()
depth_levels = depth_0.shape[0]
cell_thickness = fmesh.variables['e3' + grid.lower()][:].squeeze()
fmesh.close()
# extract the lat, lon coordinates of the boundary from the bdy_coordinate file
# bdy_i, bdy_j are the grid indices of the boundary points
# bdy_lat, bdy_lon are the coorindates of the boundary points
# bdy_r is the "r" index in the bdy, i.e. the ordinal number of the bdy layer (generally the baoundary is imposed on a band along the bpoundary that is R cells thick
fcoords = DS(coordsfile)
BDY['nbj'] = fcoords.variables['nbj' + grid.lower()][:].squeeze() - 1
BDY['nbi'] = fcoords.variables['nbi' + grid.lower()][:].squeeze() - 1
BDY['lat'] = fcoords.variables['gphi' + grid.lower()][:].squeeze()
BDY['lon'] = fcoords.variables['glam' + grid.lower()][:].squeeze()
BDY['nbr'] = fcoords.variables['nbr' + grid.lower()][:].squeeze()
fcoords.close()
#extract depth for all bdy points:
#bdy_depth=zeros((depth_levels,BDY['nbj'].size))
#bdy_cell_thickness=zeros((depth_levels,BDY['nbj'].size))
#for position in range(BDY['nbj'].size):
# bdy_depth[:,position]=depth_0[:,BDY['nbj'][position],BDY['nbi'][position]]
# bdy_cell_thickness[:,position]=cell_thickness[:,BDY['nbj'][position],BDY['nbi'][position]]
BDY['depth'] = depth_0.data #bdy_depth
BDY['e3'] = cell_thickness.data #bdy_cell_thickness
return BDY
def write_bdy(var_to_write,
ncname,
varname,
y0=1960,
y1=2099,
climatology=False):
# this write the BDY in a NEMO netcdf BDY file
count = -1
if climatology == False and var_to_write.shape[0] != (y1 - y0 + 1) * 12:
print(
'time size of the array does not correspondes with the dates provided'
)
print('size: ', var_to_write.shape[0], 'dates: %4i-%4i' % (y0, y1))
return
if len(var_to_write.shape) == 3:
#save 3D variable
for y in range(y0, y1 + 1):
if mod(y, 10) == 0: print('saving year %3i' % y)
for m in range(1, 13):
count = m - 1 if climatology else count + 1
ncfile = DS(ncname + '_y%4im%02i.nc' % (y, m), 'a')
ncfile.variables[varname][0, :, 0, :] = var_to_write[count]
ncfile.close()
elif len(var_to_write.shape) == 2:
# save 2D variables like barotropic velocities and ssh
for y in range(y0, y1 + 1):
if mod(y, 10) == 0: print('saving year %3i' % y)
for m in range(1, 13):
ncfile = DS(ncname + '_y%4im%02i.nc' % (y, m), 'a')
ncfile.variables[varname][0, 0, :] = var_to_write[count]
count += 1
ncfile.close()
return
def extract_values(BDY,
varfile,
varname,
scaling=1.,
bias=0.,
fill_mask=False):
# this function extracte the variable "varname" from the "varfile" netcdf file uding the BDY metadata in the BDY dictionnary
# if the surface monthly values need to be extracted then the logical switch needs to be set to true and the name of the netcdf file with the surface monthly value need to be read.
# it also extract the original depth information
# open the files and check what is the name of the depth variable
fvar = DS(varfile)
if 'lev' in fvar.variables.keys():
zvar = 'lev'
if 'nav_lev' in fvar.variables.keys():
zvar = 'nav_lev'
zdim = 'z'
elif 'depth' in fvar.variables.keys():
zvar = 'depth'
zdim = 'depth'
else:
print('No recognised depth array in the file')
zvar = ''
#extract variable
# if the length of the time dimension is big (i.e. monthly), then extract data one timstep at time to save memory
print('extracting values for ' + varname)
timelen = len(fvar.dimensions['t'])
if zvar != '':
nlev = len(fvar.dimensions[zdim])
var = zeros((timelen, nlev, BDY['nbj'].size))
else:
var = zeros((timelen, BDY['nbj'].size))
print('given the big size the extraction is one timestep at time')
print('this might take few minutes')
for time in range(timelen):
tmp_var = fvar.variables[varname][time]
if zvar != '':
var[time] = tmp_var[:, BDY['nbj'], BDY['nbi']] * scaling - bias
else:
var[time] = tmp_var[BDY['nbj'], BDY['nbi']] * scaling - bias
if mod(time, 50) == 0:
print('%4i out of %4i' % (time, timelen))
# extract depth information
if zvar != '':
original_depth = fvar.variables[zvar][:]
else:
original_depth = 0.
fvar.close()
return var, original_depth
import numpy as np
import matplotlib.pyplot as plt
from netCDF4 import Dataset as DS
f = DS("../scmdiag-output.nc","r")
time = f.dimensions['time']
ntime = len(time)
plt.figure( figsize=(10,6) )
cp = 1004
lat = 2.5e6
fc = 3600*24
camttend_all = fc/cp*f.variables['camstend'][:,::-1,0].T
camttend = np.mean(fc/cp*f.variables['camstend'][:,::-1,0], axis=0)
camttendcond = np.mean(fc/cp*f.variables['camstendcond'][:,::-1,0], axis=0)
camttendtranup = np.mean(fc/cp*f.variables['camstendtranup'][:,::-1,0], axis=0)
camttendtrandn = np.mean(fc/cp*f.variables['camstendtrandn'][:,::-1,0], axis=0)
camqtend_all = lat/cp*fc*f.variables['camqtend'][:,::-1,0].T
camqtend = np.mean(lat/cp*fc*f.variables['camqtend'][:,::-1,0], axis=0)
camqtendcond = np.mean(lat/cp*fc*f.variables['camqtendcond'][:,::-1,0], axis=0)
camqtendtranup = np.mean(lat/cp*fc*f.variables['camqtendtranup'][:,::-1,0], axis=0)
camqtendtrandn = np.mean(lat/cp*fc*f.variables['camqtendtrandn'][:,::-1,0], axis=0)
qcheck = f.variables['qcheck'][:,0]
def create_file(filename, config, coords, gridchar):
#this function create an empty boundary file to be used in the extract_BDY script
# it requires as input:
# filename: filename to create
# config: the dictionnary containing all configuration options associated to that file from the yaml file
# coords: a dictionnary containing the coordinates of the bdy points both in i,j,r and lon,lat
nr = coords['nbr' + gridchar].max()
dims = coords['nbi' + gridchar].shape
print('creating ' + filename)
ncfile = DS(filename, 'w')
ncfile.createDimension('time_counter', None)
timedim = 1
ncfile.createDimension('xb', dims[1])
ncfile.createDimension('yb', dims[0])
var_layers = []
for varname in coords.keys():
#create and copy coordinates variables
if varname[0] == 'n':
ncfile.createVariable(varname, 'int', ('yb', 'xb'))
ncfile.variables[varname][:] = coords[varname]
if varname[0] == 'g':
ncfile.createVariable(varname, 'f4', ('yb', 'xb'))
ncfile.variables[varname][:] = coords[varname]
for varname in config['variables'].keys():
var_layers += [
config['variables'][varname]['depth'],
]
max_depth = array(var_layers).max()
check = (var_layers == max_depth) | (var_layers == zeros(len(var_layers)))
if not (check.all()):
print("variables in %s have different numbers of z layers" % filename)
print("be sure they are either 0 or the same number")
sys.exit()
if max_depth > 0:
ncfile.createDimension('zb', max_depth)
ncfile.creation_date = datetime.now().isoformat()
ncfile.rim_width = '%iLL' % nr
for at in config.keys():
if at == 'variables':
continue
ncfile.setncattr(at, config[at])
var_config = config['variables']
for varname in var_config.keys():
if var_config[varname]['depth'] > 0:
ncfile.createVariable(varname,
'f4', ('time_counter', 'zb', 'yb', 'xb'),
zlib=True,
complevel=9,
fill_value=FV['f4'])
ncfile.variables[varname][:] = zeros(
(timedim, max_depth, dims[0], dims[1])) + FV['f4']
else:
ncfile.createVariable(varname,
'f4', ('time_counter', 'yb', 'xb'),
zlib=True,
complevel=9,
fill_value=FV['f4'])
ncfile.variables[varname][:] = zeros(
(timedim, dims[0], dims[1])) + FV['f4']
ncfile.variables[varname].grid = 'bdy' + gridchar.upper()
for at in var_config[varname].keys():
if at == 'depth':
continue
ncfile.variables[varname].setncattr(at, var_config[varname][at])
ncfile.close()
return
monthly = True
try:
daily = Yconfiguration['daily']
except:
daily = False
if daily & monthly:
print('The script has been configured to be both daily and monthly')
print(
'this is not currently allowed, if you need to generate both types, create two configuration files and run the script twice'
)
print('now only the MONTHLY files will be generated')
daily = False
for Yfile in Yconfiguration['files'].keys():
fcoords = DS(coords_file)
grid = Yconfiguration['files'][Yfile]['grid']
gridchar = grid.lower()
coords = {}
coords['nbi' + gridchar] = fcoords.variables['nbi' + gridchar][:]
coords['nbj' + gridchar] = fcoords.variables['nbj' + gridchar][:]
coords['nbr' + gridchar] = fcoords.variables['nbr' + gridchar][:]
coords['gphi' + gridchar] = fcoords.variables['gphi' + gridchar][:]
coords['glam' + gridchar] = fcoords.variables['glam' + gridchar][:]
try:
rim = min(coords['nbr' + gridchar].max(),
Yconfiguration['files'][Yfile]['rim'])
except:
rim = coords['nbr' + gridchar].max()
if rim != coords['nbr' + gridchar].max():
dims = coords['nbi' + gridchar].shape
def read_netcdf(netcdf_filename, varname, outputdir, outfile, year):
'''
Read out of data "varname" out of netcdf file.
If year==None, then all years are written to outputdirectory.
'''
# Location and name specification of netcdf files.
#pcglobwb_dir = "PCR_Routing/netCDF"
#basename_netcdf = "pcrglobwb_CRU21_30min_"
#extension_netcdf = "_yearly.nc"
#outputdir = "pcr_routing_out"
# Get the command line arguments
#year = sys.argv[1]
#varname = sys.argv[2]
#outfile = sys.argv[3]
#extensie =sys.argv[4]
#extension_netcdf = extensie + extension_netcdf
if (not os.path.isdir(outputdir)):
os.mkdir(outputdir)
#netcdf_file = os.path.join(pcglobwb_dir,basename_netcdf + varname + extension_netcdf)
output_file = os.path.join(os.path.join(outputdir, str(year)), outfile)
if (not os.path.isfile(netcdf_filename)):
print("***** ERROR *****")
print("File is not found.")
print(("Looked for file: " + netcdf_filename))
print("***** ERROR *****")
raise IOError("File " + netcdf_filename + " is not found")
# Load netcdf file
nc = DS(netcdf_filename)
# Check whether varname is in the netcdf file
if (not varname in list(nc.variables.keys())):
# There is a problem, varname is not available.
print((varname + " is not found in file: " + netcdf_filename))
print(("Parameters available: " +
",".join(map(str, list(nc.variables.keys())))))
raise IOError("Parameter " + varname + " is not found in file " +
netcdf_filename)
# Get the time settings of the nc file
try:
cdftime = utime(nc.variables['time'].units,
nc.variables['time'].calendar)
start_year = cdftime.num2date(nc.variables['time'][0]).year
end_year = cdftime.num2date(nc.variables['time'][-1]).year
except AttributeError:
# File is not time dependent.
# Close netcdf file
nc.close()
read_netcdf_constant(netcdf_filename, varname, outputdir, outfile)
if (year < start_year or year > end_year):
print("***** ERROR *****")
print(("Year: " + str(year) + " is not found in netcdf file " +
netcdf_filename))
print(("Years can be given between " + str(start_year) + " and " +
str(end_year)))
print("***** ERROR *****")
raise IOError("Year " + str(year) +
" is outside the year range in file " + netcdf_filename)
else:
# Find the year in the netcdf file
for i in range(len(nc.variables['time'])):
if (cdftime.num2date(nc.variables['time'][i]).year == int(year)):
iyear = i
break
else:
print("***** ERROR *****")
print(("Specified year " + str(year) +
" is not found in netcdf file " + netcdf_filename))
print("Years available: ")
for i in range(len(cdftime.num2date(nc.variables['time']))):
print(cdftime.num2date(nc.variables['time'][i]).year)
raise IOError("Year " + str(int(year)) + " is not found in file " +
netcdf_filename)
# Get the variable.
nc_var = nc.variables[varname][iyear, :, :]
lat = nc.variables["latitude"][:]
lon = nc.variables["longitude"][:]
nrows = len(lat)
ncols = len(lon)
# Close netcdf file
nc.close()
dim = nc_var.shape
if (dim[0] != nrows or dim[1] != ncols):
print(("File: " + netcdf_filename + " is not consistent."))
print(("In parameter file: nrows = ", nrows, " and ncols = ", ncols))
print(("In netcdf file: nrows = ", dim[0], " and ncols = ", dim[1]))
# General settings for ascii grid format
ncols = 720
nrows = 360
xll = -180
yll = -90
cellsize = 0.5
nodata_value = -9999
# Create a raster without a mask and filled with nodata.
out_grid = ascraster.Asciigrid(ncols=ncols,nrows=nrows,\
xllcorner=xll, \
yllcorner = yll, \
cellsize = cellsize,\
nodata_value = nodata_value,\
numtype=float)
# Put the nc_var data into an ascraster object
for i in range(nrows):
for j in range(ncols):
# Check whether there is no data
if (type(nc_var[i, j]) != numpy.ma.core.MaskedConstant):
icell = out_grid.get_index_from_coordin(lon[j], lat[i])
out_grid.set_data(icell, nc_var[i, j])
if (not os.path.isdir(os.path.join(outputdir, year))):
os.mkdir(os.path.join(outputdir, year))
out_grid.write_ascii_file(output_file)
def read_netcdf_constant(netcdf_filename, varname, outputdir, outfile):
'''
Read out of data "varname" out of netcdf file.
If year==None, then all years are written to outputdirectory.
'''
if (not os.path.isdir(outputdir)):
os.mkdir(outputdir)
# Make output file
output_file = os.path.join(outputdir, outfile)
if (not os.path.isfile(netcdf_filename)):
print("***** ERROR *****")
print("File is not found.")
print(("Looked for file: " + netcdf_filename))
print("***** ERROR *****")
raise IOError("File " + netcdf_filename + " is not found")
# Load netcdf file
nc = DS(netcdf_filename)
# Check whether varname is in the netcdf file
if (not varname in list(nc.variables.keys())):
# There is a problem, varname is not available.
print((varname + " is not found in file: " + netcdf_filename))
print(("Parameters available: " +
",".join(map(str, list(nc.variables.keys())))))
raise IOError("Parameter " + varname + " is not found in file " +
netcdf_filename)
# Get the variable.
nc_var = nc.variables[varname][:, :]
try:
lat = nc.variables["latitude"][:]
except KeyError:
lat = nc.variables["lat"][:]
try:
lon = nc.variables["longitude"][:]
except KeyError:
lon = nc.variables["lon"][:]
nrows = len(lat)
ncols = len(lon)
# Close netcdf file
nc.close()
dim = nc_var.shape
if (dim[0] != nrows or dim[1] != ncols):
print(("File: " + netcdf_filename + " is not consistent."))
print(("In parameter file: nrows = ", nrows, " and ncols = ", ncols))
print(("In netcdf file: nrows = ", dim[0], " and ncols = ", dim[1]))
# General settings for ascii grid format
ncols = 720
nrows = 360
xll = -180
yll = -90
cellsize = 0.5
nodata_value = -9999
# Create a raster without a mask and filled with nodata.
out_grid = ascraster.Asciigrid(ncols=ncols,nrows=nrows,\
xllcorner=xll, \
yllcorner = yll, \
cellsize = cellsize,\
nodata_value = nodata_value,\
numtype=float)
# Put the nc_var data into an ascraster object
for i in range(nrows):
for j in range(ncols):
# Check whether there is no data
if (type(nc_var[i, j]) != numpy.ma.core.MaskedConstant):
icell = out_grid.get_index_from_coordin(lon[j], lat[i])
out_grid.set_data(icell, nc_var[i, j])
if (not os.path.isdir(os.path.join(outputdir, year))):
os.mkdir(os.path.join(outputdir, year))
out_grid.write_ascii_file(output_file)
}
#here the information about themetadata are read and sotred tin the BDY dictionnary
BDY = define_bdy(coordsfile, meshfile, grid)
for varname in variables.keys():
var_keys = Yconfiguration['variables'][varname]
print('Processing %s' % varname)
if 'fixed_value' in var_keys.keys():
outfile = var_keys['output_file']
outvar = out_dict[varname]
value = var_keys['fixed_value']
print('Applying fixed value of %s' % str(value))
tmp_file = DS(outfile + '_y%4im%02i.nc' % (ystart, 1))
tmp_dims = tmp_file.variables[outvar][:].shape
tmp_file.close()
output_val = value * ones(
((yend - ystart + 1) * 12, tmp_dims[1], tmp_dims[3]))
write_bdy(output_val, outfile, outvar, y0=ystart, y1=yend)
continue
elif 'input_file' in var_keys.keys():
# extract the BGC variable from the reference dataset
print('Extracting field from %s' % var_keys['input_file'])
output_val, input_depth = extract_values(BDY,
var_keys['input_file'],
infile_dict[varname])
climatology = True if 'climatology' in var_keys else False