def execute(tasks):
global log, time_axes_, depth_axes_, table_root_
log.info("Looking up variables in files...")
tasks = lookup_variables(tasks)
log.info("Creating NEMO grids in CMOR...")
create_grids(tasks)
log.info("Executing %d NEMO tasks..." % len(tasks))
log.info("Cmorizing NEMO tasks...")
task_groups = cmor_utils.group(tasks, lambda tsk: getattr(tsk, cmor_task.output_path_key, None))
for filename, task_group in task_groups.iteritems():
dataset = netCDF4.Dataset(filename, 'r')
task_sub_groups = cmor_utils.group(task_group, lambda tsk: tsk.target.table)
for table, task_list in task_sub_groups.iteritems():
try:
tab_id = cmor.load_table("_".join([table_root_, table]) + ".json")
cmor.set_table(tab_id)
except Exception as e:
log.error("CMOR failed to load table %s, skipping variables %s. Reason: %s"
% (table, ','.join([tsk.target.variable for tsk in task_list]), e.message))
continue
if table not in time_axes_:
log.info("Creating time axes for table %s..." % table)
create_time_axes(dataset, task_list, table)
if table not in depth_axes_:
log.info("Creating depth axes for table %s..." % table)
create_depth_axes(dataset, task_list, table)
for task in task_list:
execute_netcdf_task(dataset, task)
dataset.close()
def get_ps_tasks(tasks):
""" find ps (surface preseure) tasks for different tables
Args:
tasks (list): list of tasks
Returns:
result (dictionary): dictionary based on the frequencies of different tasks with corresponding ps-tasks as values.
"""
global exp_name_,path_
tasks_by_freq = cmor_utils.group(tasks, lambda task: task.target.frequency)
result = {}
for freq, task_group in tasks_by_freq.iteritems():
tasks3d = [t for t in task_group if ("alevel" in getattr(t.target, cmor_target.dims_key).split() or "plev19" in getattr(t.target, cmor_target.dims_key).split() or
"alevhalf" in getattr(t.target, cmor_target.dims_key).split() or "plev39" in getattr(t.target, cmor_target.dims_key).split() )]
if not any(tasks3d):
continue
ps_tasks = [t for t in task_group if t.source.variable() == "ps" and
getattr(t, "time_operator", "point") in ["mean", "point"]]
ps_task = ps_tasks[0] if any(ps_tasks) else None
if ps_task:
result[freq]=ps_task
else:
source = cmor_source.tm5_source("ps")
ps_task = cmor_task.cmor_task(source, cmor_target.cmor_target("ps", freq))
setattr(ps_task.target, cmor_target.freq_key, freq)
setattr(ps_task.target, "time_operator", ["point"])
freqid=set_freqid(freq)
filepath=cmor_utils.find_tm5_output(path_,exp_name_,"ps",freqid)
setattr(ps_task, cmor_task.output_path_key, filepath[0])
result[freq]=ps_task
for task3d in tasks3d:
setattr(task3d, "ps_task", ps_task)
return result
def get_sp_tasks(tasks, autofilter):
global ifs_spectral_file_
tasks_by_freq = cmor_utils.group(tasks, lambda task: task.target.frequency)
result = []
for freq, task_group in tasks_by_freq.iteritems():
tasks3d = [
t for t in task_group
if "alevel" in getattr(t.target, cmor_target.dims_key).split()
]
if not any(tasks3d):
continue
surf_pressure_tasks = [
t for t in task_group
if t.source.get_grib_code() == surface_pressure
and getattr(t, "time_operator", "point") in ["mean", "point"]
]
surf_pressure_task = surf_pressure_tasks[0] if any(
surf_pressure_tasks) else None
if surf_pressure_task:
result.append(surf_pressure_task)
else:
source = cmor_source.tm5_source(surface_pressure)
surf_pressure_task = cmor_task.cmor_task(
source, cmor_target.cmor_target("ps", freq))
setattr(surf_pressure_task.target, cmor_target.freq_key, freq)
setattr(surf_pressure_task.target, "time_operator", ["point"])
find_sp_variable(surf_pressure_task, autofilter)
result.append(surf_pressure_task)
for task3d in tasks3d:
setattr(task3d, "sp_task", surf_pressure_task)
return result
def execute(tasks):
global log, time_axes_, depth_axes_, table_root_, nemo_files_
log.info("Executing %d NEMO tasks..." % len(tasks))
log.info("Cmorizing NEMO tasks...")
taskdict = cmor_utils.group(tasks, lambda t: t.target.table)
for k, v in taskdict.iteritems():
tab = k
tskgroup = v
freq = tskgroup[0].target.frequency
files = select_freq_files(freq)
targetvars = [t.target.variable for t in tskgroup]
if (len(files) == 0):
log.error(
"No NEMO output files found for frequency %s in file list %s, skipping variables %s"
% (freq, str(nemo_files_), str(targetvars)))
continue
log.info("Loading CMOR table %s to process %d variables..." %
(tab, len(targetvars)))
tab_id = -1
try:
tab_id = cmor.load_table("_".join([table_root_, tab]) + ".json")
cmor.set_table(tab_id)
except:
log.error("CMOR failed to load table %s, skipping variables %s" %
(tab, str(targetvars)))
continue
log.info("Creating time axes for table %s..." % tab)
time_axes_[tab_id] = create_time_axis(freq, files)
log.info("Creating depth axes for table %s..." % tab)
if (not tab_id in depth_axes_):
depth_axes_[tab_id] = create_depth_axes(tab_id, files)
taskmask = dict([t, False] for t in tskgroup)
# Loop over files:
for ncf in files:
try:
ds = netCDF4.Dataset(ncf, 'r')
for task in tskgroup:
if (task.source.var() in ds.variables):
if (taskmask[task]):
log.warning(
"Ignoring source variable in nc file %s, since it has already been cmorized."
% ncf)
else:
log.info(
"Cmorizing source variable %s to target variable %s..."
% (task.source.var_id, task.target.variable))
execute_netcdf_task(task, ds, tab_id)
taskmask[task] = True
finally:
ds.close()
for task, executed in taskmask.iteritems():
if (not executed):
log.error(
"The source variable %s could not be found in the input NEMO data"
% task.source.var_id)
def create_grids(tasks):
global grid_ids_
cmor.load_table(table_root_ + "_grids.json")
task_groups = cmor_utils.group(tasks, lambda t: getattr(t, cmor_task.output_path_key, None))
for filename, task_list in task_groups.iteritems():
if filename is not None:
grid = read_grid(filename)
grid_id = write_grid(grid)
grid_ids_[grid.name] = grid_id
for task in task_list:
setattr(task, "grid_id", grid_id)
def create_grids(tasks):
task_by_file = cmor_utils.group(
tasks, lambda tsk: getattr(tsk, cmor_task.output_path_key, None))
def get_nemo_grid(f):
if f == bathy_file_:
return bathy_grid_
if f == basin_file_:
return basin_grid_
return cmor_utils.get_nemo_grid(f)
file_by_grid = cmor_utils.group(task_by_file.keys(), get_nemo_grid)
for grid_name, file_paths in file_by_grid.iteritems():
output_files = set(file_paths) - {bathy_file_, basin_file_, None}
if any(output_files):
filename = list(output_files)[0]
else:
filename = file_paths[0]
log.warning(
"Using the file %s of EC-Earth to build %s due to lack of other output"
% (filename, grid_name))
grid = read_grid(filename)
write_grid(grid,
[t for fname in file_paths for t in task_by_file[fname]])
def execute(tasks):
"""execute the cmorization tasks for TM5
Description:
Args:
tasks (list): list of tasks
Returns:
boolean: success
"""
global log,time_axes_,depth_axes_,table_root_,tm5_files_,areacella_,using_grid_,ps_tasks
log.info("Executing %d tm5 tasks..." % len(tasks))
log.info("Cmorizing tm5 tasks...")
#Assign file to each task
for task in tasks:
setattr(task,cmor_task.output_path_key,None)
if task.target.frequency=='fx':
log.info('fx frequency has no variables from TM5')
task.set_failed()
continue
elif task.target.frequency=='monC':
if 'Clim' in task.target.variable:
log.info('Variable %s in table %s is climatological variable and thus not available in TM5.'%(task.target.variable,task.target.table))
task.set_failed()
continue
elif task.target.table=='Amon' and (task.target.variable=='pfull' or task.target.variable=='phalf'):
task.set_failed()
log.info('Variable %s in table %s will be produced by IFS'%(task.target.variable,task.target.table))
continue
elif task.target.frequency in ignore_frequency:
log.info('frequency %s ignored, no data prduced at this frequency'%task.target.frequency)
continue
elif 'Clim' in task.target.variable:
log.infor("Climatological variables not supported")
task.set_failed()
continue
success,freqid=check_freqid(task)
if not success:
task.set_failed()
log.info('Frequency %s for task %s not available.'(task.target.frequency,task.target.variable))
continue
for fstr in tm5_files_:
# only select files which start with variable name and have _ behind (e.g. o3 .neq. o3loss)
# and freqid has _ behing (e.g. monZ .neq. mon)
# catch variablename + '_' to prevent o3 and o3loss mixing up...
if os.path.basename(fstr).startswith(task.source.variable()+"_") and freqid+'_' in fstr :
fname=fstr
if getattr(task,cmor_task.output_path_key) == None:
setattr(task,cmor_task.output_path_key,fstr)
else:
log.critical('Second file with same frequency and name. Currently supporting only one year per directory.')
log.critical(fstr)
exit(' Exiting ece2cmor.')
if not os.path.exists(fstr):
log.info('No path found for variable %s from TM5'%(task.target.variable))
task.set_failed()
continue
ps_tasks=get_ps_tasks(tasks)
#group the taks according to table
taskdict = cmor_utils.group(tasks,lambda t:t.target.table)
for table,tasklist in taskdict.iteritems():
try:
log.info("Loading CMOR table %s to process %d variables..." % (table,len(tasklist)))
tab_id = cmor.load_table("_".join([table_root_, table]) + ".json")
cmor.set_table(tab_id)
except Exception as e:
log.error("ERR -6: CMOR failed to load table %s, skipping variables %s. Reason: %s"
% (table, ','.join([tsk.target.variable for tsk in tasklist]), e.message))
continue
# #postprocess data to zonal mean and plev39, or do it before this point
if table == '3hr' :
for task in tasklist:
task.set_failed()
log.error("Table %s will not be implemented for TM5" %(table))
log.error("ERR -6: Skipping variable %s not implemented" %(task.target.variable))
continue
#postprocess data to zonal mean and plev39, or do it before this point
if table== 'Eday':
log.info("Table Eday not supported for variable %s "%(task.target.variable))
log.info("Creating longitude and latitude axes for table %s..." % table)
dim_ids_['lat']=create_lat()
dim_ids_['lon']=create_lon()
# create or assign time axes to tasks
log.info("Creating time axes for table %s..." % table)
#create_time_axes(tasklist)
time_axes_=create_time_axes(tasklist)#time_axes
taskmask = dict([t,False] for t in tasklist)
for task in tasklist:
#define task properties
#2D grid
if task.target.variable=='ch4Clim' or task.target.variable=='ch4globalClim' or task.target.variable=='o3Clim':
log.error('ERR -8: Task for %s is not produced in any of the simulations with TM5.'%task.target.variable)
task.set_failed()
continue
ncf=getattr(task,cmor_task.output_path_key)
tgtdims = getattr(task.target, cmor_target.dims_key).split()
if "latitude" in tgtdims and "longitude" in tgtdims:
setattr(task, 'lon', dim_ids_['lon'])
setattr(task, 'lat', dim_ids_['lat'])
#ZONAL
if "latitude" in tgtdims and not "longitude" in tgtdims:
setattr(task, "zonal", True)
if "site" in tgtdims:
log.critical('Z-dimension site not implemented ')
task.set_failed()
continue
if task.status==cmor_task.status_failed:
continue
create_depth_axes(task)
if 'lambda550nm' in tgtdims :
success=create_type_axes(task)
if not success:
log.error('Lambda 550nm could not be created, setting task failed')
task.set_failed()
continue
if(taskmask[task] ):
log.warning("Ignoring source variable in nc file %s, since it has already been cmorized." % ncf)
else:
if task.status not in [cmor_task.status_failed]:
log.info("Cmorizing source variable %s to target variable %s from file %s." % (task.source.variable(),task.target.variable,ncf))
execute_netcdf_task(task,tab_id)
if task.status<0:
if task.target.variable=='cdnc':
log.error("ERR -10: Cmorizing failed for %s, but variable is produced by IFS." % (task.target.variable))
elif task.target.variable=='o3Clim':
log.error("ERR -11: Cmorizing failed for %s, check tm5par.json since source will be o3 instead of %s." % (task.target.variable, task.source.variable()))
elif task.target.variable=='phalf':
log.error("ERR -11: Cmorizing failed for %s, but variable is produced by IFS." % (task.target.variable))
elif task.target.variable=='ch4Clim' or task.target.variable=='ch4global' or task.target.variable=='ch4globalClim':
log.error("ERR -12: Cmorizing failed for %s, check tm5par.json since source will be ch4 instead of %s." % (task.target.variable, task.source.variable()))
else:
log.error("ERR -13: Cmorizing failed for %s" % (task.target.variable))
else:
taskmask[task] = True
else:
log.info("Skipping variable %s for unknown reason..." % (task.source.variable()))
for task,executed in taskmask.iteritems():
if(not executed):
log.error("ERR -14: The source variable %s of target %s in table %s failed to cmorize" % (task.source.variable(),task.target.variable,task.target.table))
failed.append([task.target.variable,task.target.table])
if len(unit_miss_match)>0:
log.info('Unit problems: %s'% unit_miss_match)
if len(failed)>0:
for ifail in failed:
log.info('Cmorization failed for : %s'%ifail)
def execute(tasks):
global log, time_axes_, depth_axes_, table_root_, tm5_files_
log.info("Executing %d tm5 tasks..." % len(tasks))
log.info("Cmorizing tm5 tasks...")
#Assign file to each task
for task in tasks:
#fname=None
setattr(task, cmor_task.output_path_key, None)
print task.target.variable
for fstr in tm5_files_:
if task.target.variable in fstr and task.target.frequency in fstr:
fname = fstr
print fname
setattr(task, cmor_task.output_path_key, fstr)
if task.target.variable == 'ps':
if task.target.frequency not in ps_tasks:
ps_tasks[task.target.frequency] = task
#temporary solution
# change to reading from nc-file
xsize = 120
xfirst = 0
yvals = numpy.linspace(-89, 89, 90)
grid = create_lonlat_grid(xsize, xfirst, yvals)
grid_ids_['lonlat'] = grid
####
cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
cmor.load_table(table_root_ + "_grids.json")
for task in tasks:
tgtdims = getattr(task.target, cmor_target.dims_key).split()
if "latitude" in tgtdims and "longitude" in tgtdims:
print 'gridid'
setattr(task, "grid_id", grid)
if "alevel" in tgtdims:
setattr(task, "ps_task", ps_tasks[task.target.frequency])
taskdict = cmor_utils.group(tasks, lambda t: t.target.table)
for k, v in taskdict.iteritems():
tab = k
tskgroup = v
freq = tskgroup[0].target.frequency
files = select_freq_files(freq)
targetvars = [t.target.variable for t in tskgroup]
if (len(files) == 0):
log.error(
"No tm5 output files found for frequency %s in file list %s, skipping variables %s"
% (freq, str(tm5_files_), str(targetvars)))
continue
log.info("Loading CMOR table %s to process %d variables..." %
(tab, len(targetvars)))
tab_id = -1
try:
tab_id = cmor.load_table("_".join([table_root_, tab]) + ".json")
cmor.set_table(tab_id)
except:
log.error("CMOR failed to load table %s, skipping variables %s" %
(tab, str(targetvars)))
continue
log.info("Creating time axes for table %s..." % tab)
create_time_axes(tskgroup)
log.info("Creating vertical axis for table %s..." % tab)
create_depth_axes(tskgroup)
taskmask = dict([t, False] for t in tskgroup)
# Loop over files:
# for ncf in files:
# try:
# ds = netCDF4.Dataset(ncf,'r')
for task in tskgroup:
if (task.source.var() in ds.variables):
if (taskmask[task]):
log.warning(
"Ignoring source variable in nc file %s, since it has already been cmorized."
% ncf)
else:
log.info(
"Cmorizing source variable %s to target variable %s..."
% (task.source.var_id, task.target.variable))
execute_netcdf_task(task, tab_id)
taskmask[task] = True
# finally:
# ds.close()
for task, executed in taskmask.iteritems():
if (not executed):
log.error(
"The source variable %s could not be found in the input tm5 data"
% task.source.var_id)