def execute_single_task(dataset, task):
global log
task.status = cmor_task.status_cmorizing
lon_axis = [] if not hasattr(task, "longitude_axis") else [getattr(task, "longitude_axis")]
lat_axis = [] if not hasattr(task, "latitude_axis") else [getattr(task, "latitude_axis")]
t_axis = [] if not hasattr(task, "time_axis") else [getattr(task, "time_axis")]
lu_axis = [] if not hasattr(task, "landUse_axis") else [getattr(task, "landUse_axis")]
veg_axis = [] if not hasattr(task, "vegtype_axis") else [getattr(task, "vegtype_axis")]
sdep_axis = [] if not hasattr(task, "sdepth_axis") else [getattr(task, "sdepth_axis")]
# loop over potential singleton axes
singleton_axis = []
for ax in dir(task):
if ax.startswith("singleton_"):
singleton_axis += [getattr(task, ax)]
axes = lon_axis + lat_axis + lu_axis + veg_axis + sdep_axis + t_axis + singleton_axis
varid = create_cmor_variable(task, dataset, axes)
ncvar = dataset.variables[task.target.out_name]
missval = getattr(ncvar, "missing_value", getattr(ncvar, "fill_value", np.nan))
factor = get_conversion_factor(getattr(task, cmor_task.conversion_key, None))
log.info("CMORizing variable %s in table %s form %s in "
"file %s..." % (task.target.out_name, task.target.table, task.source.variable(),
getattr(task, cmor_task.output_path_key)))
cmor_utils.netcdf2cmor(varid, ncvar, 0, factor, missval=getattr(task.target, cmor_target.missval_key, missval),
swaplatlon=True)
closed_file = cmor.close(varid, file_name=True)
log.info("CMOR closed file %s" % closed_file)
task.status = cmor_task.status_cmorized
def execute_netcdf_task(task):
global log
task.next_state()
filepath = getattr(task, cmor_task.output_path_key, None)
if not filepath:
log.error(
"Could not find file containing data for variable %s in table %s" %
(task.target.variable, task.target.table))
return
store_var = getattr(task, "store_with", None)
surf_pressure_task = getattr(task, "sp_task", None)
surf_pressure_path = getattr(surf_pressure_task, "path",
None) if surf_pressure_task else None
if store_var and not surf_pressure_path:
log.error(
"Could not find file containing surface pressure for model level variable...skipping variable %s in table "
"%s" % (task.target.variable, task.target.table))
return
axes = []
grid_id = getattr(task, "grid_id", 0)
if grid_id != 0:
axes.append(grid_id)
if hasattr(task, "z_axis_id"):
axes.append(getattr(task, "z_axis_id"))
time_id = getattr(task, "time_axis", 0)
if time_id != 0:
axes.append(time_id)
try:
dataset = netCDF4.Dataset(filepath, 'r')
except Exception as e:
log.error(
"Could not read netcdf file %s while cmorizing variable %s in table %s. Cause: %s"
% (filepath, task.target.variable, task.target.table, e.message))
return
try:
ncvars = dataset.variables
codestr = str(task.source.get_grib_code().var_id)
varlist = [
v for v in ncvars
if str(getattr(ncvars[v], "code", None)) == codestr
]
if len(varlist) == 0:
varlist = [v for v in ncvars if str(v) == "var" + codestr]
if len(varlist) > 1:
log.warning(
"CDO variable retrieval resulted in multiple (%d) netcdf variables; will take first"
% len(varlist))
ncvar = ncvars[varlist[0]]
unit = getattr(ncvar, "units", None)
if (not unit) or hasattr(task, cmor_task.conversion_key):
unit = getattr(task.target, "units")
if len(getattr(task.target, "positive", "")) > 0:
var_id = cmor.variable(table_entry=str(task.target.variable),
units=str(unit),
axis_ids=axes,
positive="down")
else:
var_id = cmor.variable(table_entry=str(task.target.variable),
units=str(unit),
axis_ids=axes)
flip_sign = (getattr(task.target, "positive", None) == "up")
factor = get_conversion_factor(
getattr(task, cmor_task.conversion_key, None),
getattr(task, cmor_task.output_frequency_key))
time_dim, index = -1, 0
for d in ncvar.dimensions:
if d.startswith("time"):
time_dim = index
break
index += 1
mask = getattr(task.target, cmor_target.mask_key, None)
mask_array = masks[mask].get("array", None) if mask in masks else None
missval = getattr(task.target, cmor_target.missval_key, 1.e+20)
if flip_sign:
missval = -missval
cmor_utils.netcdf2cmor(var_id,
ncvar,
time_dim,
factor,
store_var,
get_sp_var(surf_pressure_path),
swaplatlon=False,
fliplat=True,
mask=mask_array,
missval=missval)
cmor.close(var_id)
task.next_state()
if store_var:
cmor.close(store_var)
finally:
dataset.close()
def execute_netcdf_task(task):
global log
task.next_state()
filepath = getattr(task, cmor_task.output_path_key, None)
if not filepath:
log.error(
"Could not find file containing data for variable %s in table %s" %
(task.target.variable, task.target.table))
return
store_var = getattr(task, "store_with", None)
surf_pressure_task = getattr(task, "sp_task", None)
surf_pressure_path = getattr(surf_pressure_task, "path",
None) if surf_pressure_task else None
if store_var and not surf_pressure_path:
log.error(
"Could not find file containing surface pressure for model level variable...skipping variable %s in table "
"%s" % (task.target.variable, task.target.table))
return
axes = []
t_bnds = []
if hasattr(task, "grid_id"):
task_grid_id = getattr(task, "grid_id")
if isinstance(task_grid_id, tuple):
axes.extend([a for a in task_grid_id if a is not None])
else:
axes.append(task_grid_id)
if hasattr(task, "z_axis_id"):
axes.append(getattr(task, "z_axis_id"))
if hasattr(task, "t_axis_id"):
axes.append(getattr(task, "t_axis_id"))
t_bnds = time_axis_bnds.get(getattr(task, "t_axis_id"), [])
try:
dataset = netCDF4.Dataset(filepath, 'r')
except Exception as e:
log.error(
"Could not read netcdf file %s while cmorizing variable %s in table %s. Cause: %s"
% (filepath, task.target.variable, task.target.table, e.message))
return
try:
ncvars = dataset.variables
dataset.set_auto_mask(False)
codestr = str(task.source.get_grib_code().var_id)
varlist = [
v for v in ncvars
if str(getattr(ncvars[v], "code", None)) == codestr
]
if len(varlist) == 0:
varlist = [v for v in ncvars if str(v) == "var" + codestr]
if task.target.variable == "areacella":
varlist = ["cell_area"]
if len(varlist) == 0:
log.error(
"No suitable variable found in cdo-produced file %s fro cmorizing variable %s in table %s... "
"dismissing task" %
(filepath, task.target.variable, task.target.table))
task.set_failed()
return
if len(varlist) > 1:
log.warning(
"CDO variable retrieval resulted in multiple (%d) netcdf variables; will take first"
% len(varlist))
ncvar = ncvars[varlist[0]]
unit = getattr(ncvar, "units", None)
if (not unit) or hasattr(task, cmor_task.conversion_key):
unit = getattr(task.target, "units")
if len(getattr(task.target, "positive", "")) > 0:
var_id = cmor.variable(table_entry=str(task.target.variable),
units=str(unit),
axis_ids=axes,
positive="down")
else:
var_id = cmor.variable(table_entry=str(task.target.variable),
units=str(unit),
axis_ids=axes)
flip_sign = (getattr(task.target, "positive", None) == "up")
factor, term = get_conversion_constants(
getattr(task, cmor_task.conversion_key, None),
getattr(task, cmor_task.output_frequency_key))
time_dim, index = -1, 0
for d in ncvar.dimensions:
if d.startswith("time"):
time_dim = index
break
index += 1
time_selection = None
time_stamps = cmor_utils.read_time_stamps(filepath)
if any(time_stamps) and len(t_bnds) > 0:
time_slice_map = []
for bnd in t_bnds:
candidates = [t for t in time_stamps if bnd[0] <= t <= bnd[1]]
if any(candidates):
time_slice_map.append(time_stamps.index(candidates[0]))
else:
log.warning(
"For variable %s in table %s, no valid time point could be found at %s...inserting "
"missing values" %
(task.target.variable, task.target.table, str(bnd[0])))
time_slice_map.append(-1)
time_selection = numpy.array(time_slice_map)
mask = getattr(task.target, cmor_target.mask_key, None)
mask_array = masks[mask].get("array", None) if mask in masks else None
missval = getattr(task.target, cmor_target.missval_key, 1.e+20)
if flip_sign:
missval = -missval
cmor_utils.netcdf2cmor(var_id,
ncvar,
time_dim,
factor,
term,
store_var,
get_sp_var(surf_pressure_path),
swaplatlon=False,
fliplat=True,
mask=mask_array,
missval=missval,
time_selection=time_selection,
force_fx=(cmor_target.get_freq(
task.target) == 0))
cmor.close(var_id)
task.next_state()
if store_var:
cmor.close(store_var)
finally:
dataset.close()