def execute_netcdf_task(dataset, task):
global log, grid_ids_, depth_axes_, time_axes_
task.status = cmor_task.status_cmorizing
grid_axes = [] if not hasattr(task, "grid_id") else [getattr(task, "grid_id")]
z_axes = getattr(task, "z_axes", [])
t_axes = [] if not hasattr(task, "time_axis") else [getattr(task, "time_axis")]
axes = grid_axes + z_axes + t_axes
for type_axis in type_axes_:
if type_axis in getattr(task.target, cmor_target.dims_key):
axes.append(type_axes_[type_axis])
varid = create_cmor_variable(task, dataset, axes)
ncvar = dataset.variables[task.source.variable()]
missval = getattr(ncvar, "missing_value", getattr(ncvar, "_FillValue", numpy.nan))
if not any(grid_axes): # Fix for global averages
vals = numpy.copy(ncvar[:, :, :])
vals[vals == missval] = numpy.nan
ncvar = numpy.mean(vals[:, :, :], axis=(1, 2))
factor = get_conversion_factor(getattr(task, cmor_task.conversion_key, None))
log.info("CMORizing variable %s in table %s form %s in "
"file %s..." % (task.target.variable, task.target.table, task.source.variable(),
getattr(task, cmor_task.output_path_key)))
cmor_utils.netcdf2cmor(varid, ncvar, 0, factor, missval=getattr(task.target, cmor_target.missval_key, missval))
closed_file = cmor.close(varid, file_name=True)
log.info("CMOR closed file %s" % closed_file)
task.status = cmor_task.status_cmorized
def execute_netcdf_task(dataset, task):
global log
task.status = cmor_task.status_cmorizing
grid_axes = [] if not hasattr(task, "grid_id") else [
getattr(task, "grid_id")
]
z_axes = getattr(task, "z_axes", [])
t_axes = [] if not hasattr(task, "time_axis") else [
getattr(task, "time_axis")
]
type_axes = [
getattr(task, dim + "_axis")
for dim in type_axes_.get(task.target.table, {}).keys()
if hasattr(task, dim + "_axis")
]
# TODO: Read axes order from netcdf file!
axes = grid_axes + z_axes + type_axes + t_axes
srcvar = task.source.variable()
if task.target.variable == "basin":
ncvar, dimensions, missval = create_basins(task.target, dataset)
else:
ncvar = dataset.variables[srcvar]
dimensions = ncvar.dimensions
missval = getattr(ncvar, "missing_value",
getattr(ncvar, "_FillValue", numpy.nan))
varid = create_cmor_variable(task, srcvar, ncvar, axes)
time_dim, index, time_sel = -1, 0, None
for d in dimensions:
if d.startswith("time"):
time_dim = index
break
index += 1
time_sel = None
if len(t_axes) > 0 > time_dim:
for d in dataset.dimensions:
if d.startswith("time"):
time_sel = range(len(d)) # ensure copying of constant fields
break
if len(grid_axes) == 0: # Fix for global averages/sums
vals = numpy.ma.masked_equal(ncvar[...], missval)
ncvar = numpy.mean(vals, axis=(1, 2))
factor, term = get_conversion_constants(
getattr(task, cmor_task.conversion_key, None))
log.info('Cmorizing variable {:20} in table {:7} in file {}'.format(
srcvar, task.target.table, getattr(task, cmor_task.output_path_key)))
mask = getattr(task.target, cmor_target.mask_key, None)
if mask is not None:
mask = nemo_masks_.get(mask, None)
cmor_utils.netcdf2cmor(varid,
ncvar,
time_dim,
factor,
term,
missval=getattr(task.target,
cmor_target.missval_key, missval),
time_selection=time_sel,
mask=mask)
cmor.close(varid, file_name=True)
task.status = cmor_task.status_cmorized
def execute_netcdf_task(task, dataset, tableid):
global log, grid_ids_, depth_axes_, time_axes_
task.status = cmor_task.status_cmorizing
dims = task.target.dims
globvar = (task.source.grid() == cmor_source.nemo_grid[
cmor_source.nemo_grid.scalar])
if (globvar):
axes = []
else:
if (not task.source.grid() in grid_ids_):
log.error(
"Grid axis for %s has not been created; skipping variable." %
task.source.grid())
return
axes = [grid_ids_[task.source.grid()]]
if ((globvar and dims == 1) or (not globvar and dims == 3)):
grid_index = cmor_source.nemo_grid.index(task.source.grid())
if (not grid_index in cmor_source.nemo_depth_axes):
log.error(
"Depth axis for grid %s has not been created; skipping variable."
% task.source.grid())
return
zaxid = depth_axes_[tableid][grid_index]
axes.append(zaxid)
axes.append(time_axes_[tableid])
for type in type_axes_:
if type in getattr(task.target, cmor_target.dims_key):
axes.append(type_axes_[type])
varid = create_cmor_variable(task, dataset, axes)
ncvar = dataset.variables[task.source.var()]
missval = getattr(ncvar, "missing_value",
getattr(ncvar, "_FillValue", numpy.nan))
if (globvar): # Fix for global averages
vals = numpy.copy(ncvar[:, :, :])
vals[vals == missval] = numpy.nan
ncvar = numpy.nanmean(vals[:, :, :], axis=(1, 2))
factor = get_conversion_factor(
getattr(task, cmor_task.conversion_key, None))
cmor_utils.netcdf2cmor(varid,
ncvar,
0,
factor,
missval=getattr(task.target,
cmor_target.missval_key, missval))
cmor.close(varid)
task.status = cmor_task.status_cmorized
def execute_netcdf_task(task,tableid):
"""excute task for netcdf data
Args:
task (cmor.task): task which will be handled
tableid (cmor.table): table which will have this task
Returns:
boolean: success of writing a variable
"""
global log,dim_ids_,depth_axes_,time_axes_,areacella_
interpolate_to_pressure=False
task.status = cmor_task.status_cmorizing
filepath = getattr(task, cmor_task.output_path_key, None)
if not filepath:
log.error("ERR -15: Could not find file containing data for variable %s in table %s" % (task.target.variable,task.target.table))
task.set_failed()
return
store_var = getattr(task, "store_with", None)
if( task.target.dims >= 3):
if ('lon' in dim_ids_ and 'lat' in dim_ids_):
axes = [dim_ids_['lat'],dim_ids_['lon']]
else:
dim_ids_['lat']=create_lat()
dim_ids_['lon']=create_lon()
axes=[dim_ids_['lat'],dim_ids_['lon']]
if hasattr(task, "z_axis_id"):
axes.append(getattr(task, "z_axis_id"))
checkaxes=getattr(task.target, cmor_target.dims_key).split()
if 'plev19' in checkaxes:
interpolate_to_pressure=True
elif'plev39' in checkaxes:
interpolate_to_pressure=True
# make explicit if just to check that all axes are there
elif 'alevel' in checkaxes:
interpolate_to_pressure=False
elif 'alevhalf' in checkaxes:
interpolate_to_pressure=False
else:
log.error('ERR -16: unknown dimension in z_axis_id')
else:
log.error('ERR -17: No z_axis_id found.')
elif ( task.target.dims == 2):
if task.target.table=='AERmonZ':
'''
2D Zonal lat+lev
'''
#cmor.load_table(table_root_ + "_coordinate.json")
if 'lat' in dim_ids_:
axes=[dim_ids_['lat']]
else:
dim_ids_['lat']=create_lat()
axes=[dim_ids_['lat']]
# zonal variables...
#needs lat only, no grid....
if hasattr(task, "z_axis_id"):
axes.append(getattr(task, "z_axis_id"))
if 'plev19' in getattr(task.target, cmor_target.dims_key).split():
interpolate_to_pressure=True
elif'plev39' in getattr(task.target, cmor_target.dims_key).split():
interpolate_to_pressure=True
elif not hasattr(task, "z_axis_id"):
'''
2D variables lon+lat
'''
if not ('lon' in dim_ids_ and 'lat' in dim_ids_):
dim_ids_['lat']=create_lat()
dim_ids_['lon']=create_lon()
axes=[dim_ids_['lat'],dim_ids_['lon']]
else:
axes = [dim_ids_['lat'],dim_ids_['lon']]
else:
log.error('ERR -18: unsupported 2D dimensions %s'%task.target.dims)
exit('Exiting!')
elif task.target.dims==0:
axes=[]
else:
log.error('ERR -19: unsupported dimensions %s for variable'%(task.target.dims,task.target.variable))
exit()
time_id = getattr(task, "time_axis", 0)
if time_id != 0:
axes.append(time_id)
for key in type_axes_:
if key[0]==task.target.table and key[1] in getattr(task.target, cmor_target.dims_key):
axes.append(type_axes_[key])
try:
dataset = netCDF4.Dataset(filepath, 'r')
except Exception as e:
log.error("ERR -20: Could not read netcdf file %s while cmorizing variable %s in table %s. Cause: %s" % (
filepath, task.target.variable, task.target.table, e.message))
return
varid = create_cmor_variable(task,dataset,axes)
if varid <0:
return False
## for pressure level variables we need to do interpolation, for which we need
## pyngl module
if interpolate_to_pressure:
psdata=get_ps_var(getattr(getattr(task,'ps_task',None),cmor_task.output_path_key,None))
pressure_levels=getattr(task,'pressure_levels')
ncvar=interpolate_plev(pressure_levels,dataset,psdata,task.source.variable())
else:
ncvar = dataset.variables[task.source.variable()]
# handle zonal vars
if task.target.table=='AERmonZ':
# assumption: data is shape [time,lat,lon] (roll longitude dimensio
vals=numpy.copy(ncvar[:])
# zonal mean so mean over longitudes
vals=numpy.mean(vals,axis=-1)
# change shape, swap lat<->lev
vals=numpy.swapaxes(vals,1,2)
missval = getattr(ncvar,"missing_value",getattr(ncvar,"_FillValue",numpy.nan))
ncvar=vals.copy()
#handle global means
elif task.target.dims==0:
# global means
missval = getattr(ncvar,"missing_value",getattr(ncvar,"_FillValue",numpy.nan))
vals=numpy.copy(ncvar[:])
vals=numpy.mean(vals,axis=(1))
# calculate area-weighted mean
vals=numpy.sum((vals*areacella_[numpy.newaxis,:,:]),axis=(1,2))/numpy.sum(areacella_)
#handle normal case
else:# assumption: data is shape [time,lat,lon] (we need to roll longitude dimension so that
# the data corresponds to the dimension definition of tables (from [-180 , 180] to [0,360] deg).)
# so by half the longitude dimension
missval = getattr(ncvar,"missing_value",getattr(ncvar,"_FillValue",numpy.nan))
vals=numpy.copy(ncvar[:])
dims = numpy.shape(vals)
nroll=dims[-1]/2
ncvar = numpy.roll(vals,nroll,len(dims)-1)
vals=numpy.copy(ncvar[:,:,:])
# Default values
factor = 1.0
term=0.0
timdim=0
# 3D variables need the surface pressure for calculating the pressure at model levels
if store_var:
#get the ps-data associated with this data
psdata=get_ps_var(getattr(getattr(task,'ps_task',None),cmor_task.output_path_key,None))
# roll psdata like the original
psdata=numpy.roll(psdata[:],nroll,len(numpy.shape(psdata[:]))-1)
cmor_utils.netcdf2cmor(varid, ncvar, timdim, factor, term, store_var, psdata,
swaplatlon=False, fliplat=True, mask=None,missval=missval)
else:
cmor_utils.netcdf2cmor(varid, ncvar, timdim, factor, term, store_var, None,
swaplatlon=False, fliplat=True, mask=None,missval=missval)
cmor.close(varid)
if store_var:
cmor.close(store_var)
task.status = cmor_task.status_cmorized
def execute_netcdf_task(task, tableid):
global log, grid_ids_, depth_axes_, time_axes_
#if not tableid in depth_axes_:
# depth_axes_[tableid]=create_hybrid_level_axis(task)
task.status = cmor_task.status_cmorizing
filepath = getattr(task, cmor_task.output_path_key, None)
if not filepath:
log.error(
"Could not find file containing data for variable %s in table %s" %
(task.target.variable, task.target.table))
return
store_var = getattr(task, "store_with", None)
dims = task.target.dims
if (not task.source.grid() in grid_ids_):
log.error("Grid axis for %s has not been created; skipping variable." %
task.source.grid())
return
axes = [grid_ids_[task.source.grid()]]
if (dims == 3):
grid_index = cmor_source.tm5_grid.index(task.source.grid())
print cmor_source.tm5_depth_axes, grid_index, tableid, depth_axes_
if (not grid_index in cmor_source.tm5_depth_axes):
log.error(
"Depth axis for grid %s has not been created; skipping variable."
% task.source.grid())
return
#zaxid = depth_axes_[tableid][grid_index]
if hasattr(task, "z_axis_id"):
axes.append(getattr(task, "z_axis_id"))
#axes.append(zaxid)
time_id = getattr(task, "time_axis", 0)
if time_id != 0:
axes.append(time_id)
#axes.append(time_axes_[tableid])
for type in type_axes_:
if type in getattr(task.target, cmor_target.dims_key):
axes.append(type_axes_[type])
try:
dataset = netCDF4.Dataset(filepath, 'r')
except Exception as e:
log.error(
"Could not read netcdf file %s while cmorizing variable %s in table %s. Cause: %s"
% (filepath, task.target.variable, task.target.table, e.message))
return
varid = create_cmor_variable(task, dataset, axes)
ncvar = dataset.variables[task.source.var()]
vals = numpy.copy(ncvar[:])
dims = numpy.shape(vals)
nroll = dims[-1] / 2
ncvar = numpy.roll(vals, nroll, len(dims) - 1)
missval = getattr(ncvar, "missing_value",
getattr(ncvar, "_FillValue", numpy.nan))
vals = numpy.copy(ncvar[:, :, :])
#factor 1. keep it for time being
factor = 1.0 #get_conversion_factor(getattr(task,cmor_task.conversion_key,None))
cmor_utils.netcdf2cmor(varid,
ncvar,
0,
factor,
store_var,
get_ps_var(
getattr(ps_tasks[task.target.frequency],
cmor_task.output_path_key, None)),
swaplatlon=False,
fliplat=False,
mask=None,
missval=missval)
cmor.close(varid)
if store_var:
cmor.close(store_var)
task.status = cmor_task.status_cmorized