def execute(tasks): global log, time_axes_, depth_axes_, table_root_ log.info("Looking up variables in files...") tasks = lookup_variables(tasks) log.info("Creating NEMO grids in CMOR...") create_grids(tasks) log.info("Executing %d NEMO tasks..." % len(tasks)) log.info("Cmorizing NEMO tasks...") task_groups = cmor_utils.group(tasks, lambda tsk: getattr(tsk, cmor_task.output_path_key, None)) for filename, task_group in task_groups.iteritems(): dataset = netCDF4.Dataset(filename, 'r') task_sub_groups = cmor_utils.group(task_group, lambda tsk: tsk.target.table) for table, task_list in task_sub_groups.iteritems(): try: tab_id = cmor.load_table("_".join([table_root_, table]) + ".json") cmor.set_table(tab_id) except Exception as e: log.error("CMOR failed to load table %s, skipping variables %s. Reason: %s" % (table, ','.join([tsk.target.variable for tsk in task_list]), e.message)) continue if table not in time_axes_: log.info("Creating time axes for table %s..." % table) create_time_axes(dataset, task_list, table) if table not in depth_axes_: log.info("Creating depth axes for table %s..." % table) create_depth_axes(dataset, task_list, table) for task in task_list: execute_netcdf_task(dataset, task) dataset.close()
def get_ps_tasks(tasks): """ find ps (surface preseure) tasks for different tables Args: tasks (list): list of tasks Returns: result (dictionary): dictionary based on the frequencies of different tasks with corresponding ps-tasks as values. """ global exp_name_,path_ tasks_by_freq = cmor_utils.group(tasks, lambda task: task.target.frequency) result = {} for freq, task_group in tasks_by_freq.iteritems(): tasks3d = [t for t in task_group if ("alevel" in getattr(t.target, cmor_target.dims_key).split() or "plev19" in getattr(t.target, cmor_target.dims_key).split() or "alevhalf" in getattr(t.target, cmor_target.dims_key).split() or "plev39" in getattr(t.target, cmor_target.dims_key).split() )] if not any(tasks3d): continue ps_tasks = [t for t in task_group if t.source.variable() == "ps" and getattr(t, "time_operator", "point") in ["mean", "point"]] ps_task = ps_tasks[0] if any(ps_tasks) else None if ps_task: result[freq]=ps_task else: source = cmor_source.tm5_source("ps") ps_task = cmor_task.cmor_task(source, cmor_target.cmor_target("ps", freq)) setattr(ps_task.target, cmor_target.freq_key, freq) setattr(ps_task.target, "time_operator", ["point"]) freqid=set_freqid(freq) filepath=cmor_utils.find_tm5_output(path_,exp_name_,"ps",freqid) setattr(ps_task, cmor_task.output_path_key, filepath[0]) result[freq]=ps_task for task3d in tasks3d: setattr(task3d, "ps_task", ps_task) return result
def get_sp_tasks(tasks, autofilter): global ifs_spectral_file_ tasks_by_freq = cmor_utils.group(tasks, lambda task: task.target.frequency) result = [] for freq, task_group in tasks_by_freq.iteritems(): tasks3d = [ t for t in task_group if "alevel" in getattr(t.target, cmor_target.dims_key).split() ] if not any(tasks3d): continue surf_pressure_tasks = [ t for t in task_group if t.source.get_grib_code() == surface_pressure and getattr(t, "time_operator", "point") in ["mean", "point"] ] surf_pressure_task = surf_pressure_tasks[0] if any( surf_pressure_tasks) else None if surf_pressure_task: result.append(surf_pressure_task) else: source = cmor_source.tm5_source(surface_pressure) surf_pressure_task = cmor_task.cmor_task( source, cmor_target.cmor_target("ps", freq)) setattr(surf_pressure_task.target, cmor_target.freq_key, freq) setattr(surf_pressure_task.target, "time_operator", ["point"]) find_sp_variable(surf_pressure_task, autofilter) result.append(surf_pressure_task) for task3d in tasks3d: setattr(task3d, "sp_task", surf_pressure_task) return result
def execute(tasks): global log, time_axes_, depth_axes_, table_root_, nemo_files_ log.info("Executing %d NEMO tasks..." % len(tasks)) log.info("Cmorizing NEMO tasks...") taskdict = cmor_utils.group(tasks, lambda t: t.target.table) for k, v in taskdict.iteritems(): tab = k tskgroup = v freq = tskgroup[0].target.frequency files = select_freq_files(freq) targetvars = [t.target.variable for t in tskgroup] if (len(files) == 0): log.error( "No NEMO output files found for frequency %s in file list %s, skipping variables %s" % (freq, str(nemo_files_), str(targetvars))) continue log.info("Loading CMOR table %s to process %d variables..." % (tab, len(targetvars))) tab_id = -1 try: tab_id = cmor.load_table("_".join([table_root_, tab]) + ".json") cmor.set_table(tab_id) except: log.error("CMOR failed to load table %s, skipping variables %s" % (tab, str(targetvars))) continue log.info("Creating time axes for table %s..." % tab) time_axes_[tab_id] = create_time_axis(freq, files) log.info("Creating depth axes for table %s..." % tab) if (not tab_id in depth_axes_): depth_axes_[tab_id] = create_depth_axes(tab_id, files) taskmask = dict([t, False] for t in tskgroup) # Loop over files: for ncf in files: try: ds = netCDF4.Dataset(ncf, 'r') for task in tskgroup: if (task.source.var() in ds.variables): if (taskmask[task]): log.warning( "Ignoring source variable in nc file %s, since it has already been cmorized." % ncf) else: log.info( "Cmorizing source variable %s to target variable %s..." % (task.source.var_id, task.target.variable)) execute_netcdf_task(task, ds, tab_id) taskmask[task] = True finally: ds.close() for task, executed in taskmask.iteritems(): if (not executed): log.error( "The source variable %s could not be found in the input NEMO data" % task.source.var_id)
def create_grids(tasks): global grid_ids_ cmor.load_table(table_root_ + "_grids.json") task_groups = cmor_utils.group(tasks, lambda t: getattr(t, cmor_task.output_path_key, None)) for filename, task_list in task_groups.iteritems(): if filename is not None: grid = read_grid(filename) grid_id = write_grid(grid) grid_ids_[grid.name] = grid_id for task in task_list: setattr(task, "grid_id", grid_id)
def create_grids(tasks): task_by_file = cmor_utils.group( tasks, lambda tsk: getattr(tsk, cmor_task.output_path_key, None)) def get_nemo_grid(f): if f == bathy_file_: return bathy_grid_ if f == basin_file_: return basin_grid_ return cmor_utils.get_nemo_grid(f) file_by_grid = cmor_utils.group(task_by_file.keys(), get_nemo_grid) for grid_name, file_paths in file_by_grid.iteritems(): output_files = set(file_paths) - {bathy_file_, basin_file_, None} if any(output_files): filename = list(output_files)[0] else: filename = file_paths[0] log.warning( "Using the file %s of EC-Earth to build %s due to lack of other output" % (filename, grid_name)) grid = read_grid(filename) write_grid(grid, [t for fname in file_paths for t in task_by_file[fname]])
def execute(tasks): """execute the cmorization tasks for TM5 Description: Args: tasks (list): list of tasks Returns: boolean: success """ global log,time_axes_,depth_axes_,table_root_,tm5_files_,areacella_,using_grid_,ps_tasks log.info("Executing %d tm5 tasks..." % len(tasks)) log.info("Cmorizing tm5 tasks...") #Assign file to each task for task in tasks: setattr(task,cmor_task.output_path_key,None) if task.target.frequency=='fx': log.info('fx frequency has no variables from TM5') task.set_failed() continue elif task.target.frequency=='monC': if 'Clim' in task.target.variable: log.info('Variable %s in table %s is climatological variable and thus not available in TM5.'%(task.target.variable,task.target.table)) task.set_failed() continue elif task.target.table=='Amon' and (task.target.variable=='pfull' or task.target.variable=='phalf'): task.set_failed() log.info('Variable %s in table %s will be produced by IFS'%(task.target.variable,task.target.table)) continue elif task.target.frequency in ignore_frequency: log.info('frequency %s ignored, no data prduced at this frequency'%task.target.frequency) continue elif 'Clim' in task.target.variable: log.infor("Climatological variables not supported") task.set_failed() continue success,freqid=check_freqid(task) if not success: task.set_failed() log.info('Frequency %s for task %s not available.'(task.target.frequency,task.target.variable)) continue for fstr in tm5_files_: # only select files which start with variable name and have _ behind (e.g. o3 .neq. o3loss) # and freqid has _ behing (e.g. monZ .neq. mon) # catch variablename + '_' to prevent o3 and o3loss mixing up... if os.path.basename(fstr).startswith(task.source.variable()+"_") and freqid+'_' in fstr : fname=fstr if getattr(task,cmor_task.output_path_key) == None: setattr(task,cmor_task.output_path_key,fstr) else: log.critical('Second file with same frequency and name. Currently supporting only one year per directory.') log.critical(fstr) exit(' Exiting ece2cmor.') if not os.path.exists(fstr): log.info('No path found for variable %s from TM5'%(task.target.variable)) task.set_failed() continue ps_tasks=get_ps_tasks(tasks) #group the taks according to table taskdict = cmor_utils.group(tasks,lambda t:t.target.table) for table,tasklist in taskdict.iteritems(): try: log.info("Loading CMOR table %s to process %d variables..." % (table,len(tasklist))) tab_id = cmor.load_table("_".join([table_root_, table]) + ".json") cmor.set_table(tab_id) except Exception as e: log.error("ERR -6: CMOR failed to load table %s, skipping variables %s. Reason: %s" % (table, ','.join([tsk.target.variable for tsk in tasklist]), e.message)) continue # #postprocess data to zonal mean and plev39, or do it before this point if table == '3hr' : for task in tasklist: task.set_failed() log.error("Table %s will not be implemented for TM5" %(table)) log.error("ERR -6: Skipping variable %s not implemented" %(task.target.variable)) continue #postprocess data to zonal mean and plev39, or do it before this point if table== 'Eday': log.info("Table Eday not supported for variable %s "%(task.target.variable)) log.info("Creating longitude and latitude axes for table %s..." % table) dim_ids_['lat']=create_lat() dim_ids_['lon']=create_lon() # create or assign time axes to tasks log.info("Creating time axes for table %s..." % table) #create_time_axes(tasklist) time_axes_=create_time_axes(tasklist)#time_axes taskmask = dict([t,False] for t in tasklist) for task in tasklist: #define task properties #2D grid if task.target.variable=='ch4Clim' or task.target.variable=='ch4globalClim' or task.target.variable=='o3Clim': log.error('ERR -8: Task for %s is not produced in any of the simulations with TM5.'%task.target.variable) task.set_failed() continue ncf=getattr(task,cmor_task.output_path_key) tgtdims = getattr(task.target, cmor_target.dims_key).split() if "latitude" in tgtdims and "longitude" in tgtdims: setattr(task, 'lon', dim_ids_['lon']) setattr(task, 'lat', dim_ids_['lat']) #ZONAL if "latitude" in tgtdims and not "longitude" in tgtdims: setattr(task, "zonal", True) if "site" in tgtdims: log.critical('Z-dimension site not implemented ') task.set_failed() continue if task.status==cmor_task.status_failed: continue create_depth_axes(task) if 'lambda550nm' in tgtdims : success=create_type_axes(task) if not success: log.error('Lambda 550nm could not be created, setting task failed') task.set_failed() continue if(taskmask[task] ): log.warning("Ignoring source variable in nc file %s, since it has already been cmorized." % ncf) else: if task.status not in [cmor_task.status_failed]: log.info("Cmorizing source variable %s to target variable %s from file %s." % (task.source.variable(),task.target.variable,ncf)) execute_netcdf_task(task,tab_id) if task.status<0: if task.target.variable=='cdnc': log.error("ERR -10: Cmorizing failed for %s, but variable is produced by IFS." % (task.target.variable)) elif task.target.variable=='o3Clim': log.error("ERR -11: Cmorizing failed for %s, check tm5par.json since source will be o3 instead of %s." % (task.target.variable, task.source.variable())) elif task.target.variable=='phalf': log.error("ERR -11: Cmorizing failed for %s, but variable is produced by IFS." % (task.target.variable)) elif task.target.variable=='ch4Clim' or task.target.variable=='ch4global' or task.target.variable=='ch4globalClim': log.error("ERR -12: Cmorizing failed for %s, check tm5par.json since source will be ch4 instead of %s." % (task.target.variable, task.source.variable())) else: log.error("ERR -13: Cmorizing failed for %s" % (task.target.variable)) else: taskmask[task] = True else: log.info("Skipping variable %s for unknown reason..." % (task.source.variable())) for task,executed in taskmask.iteritems(): if(not executed): log.error("ERR -14: The source variable %s of target %s in table %s failed to cmorize" % (task.source.variable(),task.target.variable,task.target.table)) failed.append([task.target.variable,task.target.table]) if len(unit_miss_match)>0: log.info('Unit problems: %s'% unit_miss_match) if len(failed)>0: for ifail in failed: log.info('Cmorization failed for : %s'%ifail)
def execute(tasks): global log, time_axes_, depth_axes_, table_root_, tm5_files_ log.info("Executing %d tm5 tasks..." % len(tasks)) log.info("Cmorizing tm5 tasks...") #Assign file to each task for task in tasks: #fname=None setattr(task, cmor_task.output_path_key, None) print task.target.variable for fstr in tm5_files_: if task.target.variable in fstr and task.target.frequency in fstr: fname = fstr print fname setattr(task, cmor_task.output_path_key, fstr) if task.target.variable == 'ps': if task.target.frequency not in ps_tasks: ps_tasks[task.target.frequency] = task #temporary solution # change to reading from nc-file xsize = 120 xfirst = 0 yvals = numpy.linspace(-89, 89, 90) grid = create_lonlat_grid(xsize, xfirst, yvals) grid_ids_['lonlat'] = grid #### cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian") cmor.load_table(table_root_ + "_grids.json") for task in tasks: tgtdims = getattr(task.target, cmor_target.dims_key).split() if "latitude" in tgtdims and "longitude" in tgtdims: print 'gridid' setattr(task, "grid_id", grid) if "alevel" in tgtdims: setattr(task, "ps_task", ps_tasks[task.target.frequency]) taskdict = cmor_utils.group(tasks, lambda t: t.target.table) for k, v in taskdict.iteritems(): tab = k tskgroup = v freq = tskgroup[0].target.frequency files = select_freq_files(freq) targetvars = [t.target.variable for t in tskgroup] if (len(files) == 0): log.error( "No tm5 output files found for frequency %s in file list %s, skipping variables %s" % (freq, str(tm5_files_), str(targetvars))) continue log.info("Loading CMOR table %s to process %d variables..." % (tab, len(targetvars))) tab_id = -1 try: tab_id = cmor.load_table("_".join([table_root_, tab]) + ".json") cmor.set_table(tab_id) except: log.error("CMOR failed to load table %s, skipping variables %s" % (tab, str(targetvars))) continue log.info("Creating time axes for table %s..." % tab) create_time_axes(tskgroup) log.info("Creating vertical axis for table %s..." % tab) create_depth_axes(tskgroup) taskmask = dict([t, False] for t in tskgroup) # Loop over files: # for ncf in files: # try: # ds = netCDF4.Dataset(ncf,'r') for task in tskgroup: if (task.source.var() in ds.variables): if (taskmask[task]): log.warning( "Ignoring source variable in nc file %s, since it has already been cmorized." % ncf) else: log.info( "Cmorizing source variable %s to target variable %s..." % (task.source.var_id, task.target.variable)) execute_netcdf_task(task, tab_id) taskmask[task] = True # finally: # ds.close() for task, executed in taskmask.iteritems(): if (not executed): log.error( "The source variable %s could not be found in the input tm5 data" % task.source.var_id)