def get_levels(task, code): global log if (code.var_id, code.tab_id) == (134, 128): return grib_file.surface_level_code, [0] if 34 < code.var_id < 43 and code.tab_id == 128: return grib_file.depth_level_code, [0] if code.var_id in [139, 170, 183, 236] and code.tab_id == 128: return grib_file.depth_level_code, [0] zaxis, levels = cmor_target.get_z_axis(task.target) if zaxis is None: return grib_file.surface_level_code, [0] if zaxis in ["sdepth"]: return grib_file.depth_level_code, [0] if zaxis in ["alevel", "alevhalf"]: return grib_file.hybrid_level_code, [-1] if zaxis == "air_pressure": return grib_file.pressure_level_Pa_code, [ int(float(l)) for l in levels ] if zaxis in ["height", "altitude"]: return grib_file.height_level_code, [int( float(l)) for l in levels] # TODO: What about decimal places? log.error( "Could not convert vertical axis type %s to grib vertical coordinate " "code for %s" % (zaxis, task.target.variable)) return -1, []
def get_levels(task, code): global log # Special cases if code.tab_id == 128: gc = code.var_id if gc in [9, 134]: return grib_file.surface_level_code, [0] if gc in [35, 36, 37, 38, 39, 40, 41, 42, 139, 170, 183, 236]: return grib_file.depth_level_code, [0] if gc in [49, 165, 166]: return grib_file.height_level_code, [10] if gc in [167, 168, 201, 202]: return grib_file.height_level_code, [2] # Normal cases zaxis, levels = cmor_target.get_z_axis(task.target) if zaxis is None: return grib_file.surface_level_code, [0] if zaxis in ["sdepth"]: return grib_file.depth_level_code, [0] if zaxis in ["alevel", "alevhalf"]: return grib_file.hybrid_level_code, [-1] if zaxis == "air_pressure": return grib_file.pressure_level_Pa_code, [ int(float(level)) for level in levels ] if zaxis in ["height", "altitude"]: return grib_file.height_level_code, [ int(float(level)) for level in levels ] # TODO: What about decimal places? log.error( "Could not convert vertical axis type %s to grib vertical coordinate " "code for %s" % (zaxis, task.target.variable)) return -1, []
def test_validate_tasks(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt1 = ece2cmorlib.get_cmor_target("clwvi", "CFday") src1 = cmor_source.ifs_source.read("79.128") tsk1 = cmor_task.cmor_task(src1, tgt1) tgt2 = ece2cmorlib.get_cmor_target("ua", "Amon") src2 = cmor_source.ifs_source.read("131.128") tsk2 = cmor_task.cmor_task(src2, tgt2) valid_tasks, varstasks = grib_filter.validate_tasks([tsk1, tsk2]) assert valid_tasks == [tsk1, tsk2] key1 = (79, 128, grib_file.surface_level_code, 0, cmor_source.ifs_grid.point) key2 = (131, 128, grib_file.pressure_level_Pa_code, 92500, cmor_source.ifs_grid.spec) assert varstasks[key1] == [tsk1] assert varstasks[key2] == [tsk2] ltype, plevs = cmor_target.get_z_axis(tgt2) levs = sorted([float(p) for p in plevs]) levcheck = sorted([k[3] for k in varstasks if k[0] == 131]) assert levs == levcheck
def test_zhalfo_zdims(): abspath = get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") zhalfo = [t for t in targets if t.variable == "zhalfo"][0] assert cmor_target.get_z_axis(zhalfo)[0] == "olevhalf"
def write_ppt_files(tasks): freqgroups = cmor_utils.group(tasks, get_output_freq) for freq1 in freqgroups: for freq2 in freqgroups: if freq2 > freq1 and freq2 % freq1 == 0: freqgroups[freq2] = freqgroups[freq1] + freqgroups[freq2] for freq in freqgroups: mfp2df, mfpphy, mfp3dfs, mfp3dfp, mfp3dfv = [], [], [], [], [] alevs, plevs, hlevs = [], [], [] for task in freqgroups[freq]: zaxis, levs = cmor_target.get_z_axis(task.target) root_codes = task.source.get_root_codes() if not zaxis: for code in root_codes: if code in cmor_source.ifs_source.grib_codes_3D: log.warning( "3D grib code %s used in 2D cmor-target %s..." "assuming this is on model levels" % (str(code), task.target.variable)) mfp3dfs.append(code) elif code in cmor_source.ifs_source.grib_codes_2D_dyn: log.info( "Adding grib code %s to MFP2DF %dhr ppt file for variable " "%s in table %s" % (str(code), freq, task.target.variable, task.target.table)) mfp2df.append(code) elif code in cmor_source.ifs_source.grib_codes_2D_phy: log.info( "Adding grib code %s to MFPPHY %dhr ppt file for variable " "%s in table %s" % (str(code), freq, task.target.variable, task.target.table)) mfpphy.append(code) else: log.error("Unknown IFS grib code %s skipped" % str(code)) else: for code in root_codes: if code in cmor_source.ifs_source.grib_codes_3D: if zaxis in cmor_target.model_axes: log.info( "Adding grib code %s to MFP3DFS %dhr ppt file for variable " "%s in table %s" % (str(code), freq, task.target.variable, task.target.table)) mfp3dfs.append(code) alevs.extend(levs) elif zaxis in cmor_target.pressure_axes: log.info( "Adding grib code %s to MFP3DFP %dhr ppt file for variable " "%s in table %s" % (str(code), freq, task.target.variable, task.target.table)) mfp3dfp.append(code) plevs.extend(levs) elif zaxis in cmor_target.height_axes: log.info( "Adding grib code %s to MFP3DFV %dhr ppt file for variable " "%s in table %s" % (str(code), freq, task.target.variable, task.target.table)) mfp3dfv.append(code) hlevs.extend(levs) else: log.error( "Axis type %s unknown, adding grib code %s" "to model level variables" % (zaxis, str(code))) elif code in cmor_source.ifs_source.grib_codes_2D_dyn: mfp2df.append(code) elif code in cmor_source.ifs_source.grib_codes_2D_phy: mfpphy.append(code) else: log.error("Unknown IFS grib code %s skipped" % str(code)) # Always add the geopotential, recommended by ECMWF if cmor_source.grib_code(129) not in mfp3dfs: mfp2df.append(cmor_source.grib_code(129)) # Always add the surface pressure, recommended by ECMWF mfpphy.append(cmor_source.grib_code(134)) # Always add the logarithm of surface pressure, recommended by ECMWF mfp2df.append(cmor_source.grib_code(152)) mfp2df = sorted( list( map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp2df)))) mfpphy = sorted( list( map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfpphy)))) mfp3dfs = sorted( list( map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp3dfs)))) mfp3dfp = sorted( list( map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp3dfp)))) mfp3dfv = sorted( list( map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp3dfv)))) plevs = sorted(list(set([float(s) for s in plevs])))[::-1] hlevs = sorted(list(set([float(s) for s in hlevs]))) namelist = {"CFPFMT": "MODEL"} if any(mfp2df): namelist["NFP2DF"] = len(mfp2df) namelist["MFP2DF"] = mfp2df if any(mfpphy): namelist["NFPPHY"] = len(mfpphy) namelist["MFPPHY"] = mfpphy if any(mfp3dfs): namelist["NFP3DFS"] = len(mfp3dfs) namelist["MFP3DFS"] = mfp3dfs namelist["NRFP3S"] = -1 if any(mfp3dfp): namelist["NFP3DFP"] = len(mfp3dfp) namelist["MFP3DFP"] = mfp3dfp namelist["RFP3P"] = plevs if any(mfp3dfs): namelist["NFP3DFV"] = len(mfp3dfv) namelist["MFP3DFV"] = mfp3dfv namelist["RFP3V"] = hlevs nml = f90nml.Namelist({"NAMFPC": namelist}) nml.uppercase, nml.end_comma = True, True f90nml.write(nml, "pptdddddd%04d" % (100 * freq, ))
def get_sample_freq(task): axis, levs = cmor_target.get_z_axis(task.target) if axis in cmor_target.model_axes + cmor_target.pressure_axes + cmor_target.height_axes: return 6 else: return 3
def ifs_model_level_variable(target): zaxis, levs = cmor_target.get_z_axis(target) return zaxis not in ["alevel", "alevhalf"]
def write_ppt_files(tasks): freqgroups = cmor_utils.group(tasks, get_output_freq) # Fix for issue 313, make sure to always generate 6-hourly ppt: if freqgroups.keys() == [3]: freqgroups[6] = [] if -1 in freqgroups.keys(): freqgroups.pop(-1) freqs_to_remove = [] for freq1 in freqgroups: if freq1 <= 0: continue for freq2 in freqgroups: if freq2 > freq1: if freq2 % freq1 == 0: freqgroups[freq2] = freqgroups[freq1] + freqgroups[freq2] else: log.error("Frequency %d is not a divisor of frequency %d: this is not supported, " "removing the former" % (freq1, freq2)) freqs_to_remove.append(freq1) for freq in set(freqs_to_remove): freqgroups.pop(freq, None) num_slices_tot_sp, num_slices_tot_gp, num_blocks_tot_sp, num_blocks_tot_gp = 0, 0, 0, 0 min_freq = max(freqgroups.keys()) prev_freq = 0 fx_namelist = {} for freq in sorted(freqgroups.keys()): mfp2df, mfpphy, mfp3dfs, mfp3dfp, mfp3dfh = [], [], [], [], [] num_slices_sp, num_slices_gp, num_blocks_sp, num_blocks_gp = 0, 0, 0, 0 alevs, plevs, hlevs = [], [], [] for task in freqgroups[freq]: zaxis, levs = cmor_target.get_z_axis(task.target) root_codes = task.source.get_root_codes() if not zaxis: for code in root_codes: if freq > 0 and code in cmor_source.ifs_source.grib_codes_fx: continue if code in cmor_source.ifs_source.grib_codes_3D: # Exception for orog and areacella, depend only on lowest level of 129: if task.target.variable in ["orog", "areacella"] and code == cmor_source.grib_code(129): mfp2df.append(code) else: log.warning("3D grib code %s used in 2D cmor-target %s..." "assuming this is on model levels" % (str(code), task.target.variable)) mfp3dfs.append(code) elif code in cmor_source.ifs_source.grib_codes_2D_dyn: log.info("Adding grib code %s to MFP2DF %dhr ppt file for variable " "%s in table %s" % (str(code), freq, task.target.variable, task.target.table)) mfp2df.append(code) elif code in cmor_source.ifs_source.grib_codes_2D_phy: log.info("Adding grib code %s to MFPPHY %dhr ppt file for variable " "%s in table %s" % (str(code), freq, task.target.variable, task.target.table)) mfpphy.append(code) else: log.error("Unknown 2D IFS grib code %s skipped" % str(code)) else: for code in root_codes: if freq > 0 and code in cmor_source.ifs_source.grib_codes_fx: continue if code in cmor_source.ifs_source.grib_codes_3D: if zaxis in cmor_target.model_axes: log.info("Adding grib code %s to MFP3DFS %dhr ppt file for variable " "%s in table %s" % (str(code), freq, task.target.variable, task.target.table)) mfp3dfs.append(code) alevs.extend(levs) elif zaxis in cmor_target.pressure_axes: log.info("Adding grib code %s to MFP3DFP %dhr ppt file for variable " "%s in table %s" % (str(code), freq, task.target.variable, task.target.table)) mfp3dfp.append(code) plevs.extend(levs) elif zaxis in cmor_target.height_axes: log.info("Adding grib code %s to MFP3DFH %dhr ppt file for variable " "%s in table %s" % (str(code), freq, task.target.variable, task.target.table)) mfp3dfh.append(code) hlevs.extend(levs) else: log.error("Axis type %s unknown, adding grib code %s" "to model level variables" % (zaxis, str(code))) elif code in cmor_source.ifs_source.grib_codes_2D_dyn: mfp2df.append(code) elif code in cmor_source.ifs_source.grib_codes_2D_phy: mfpphy.append(code) # case for PEXTRA tendencies is missing else: log.error("Unknown 3D IFS grib code %s skipped" % str(code)) # Always add the geopotential, recommended by ECMWF if cmor_source.grib_code(129) not in mfp3dfs: mfp2df.append(cmor_source.grib_code(129)) # Always add the surface pressure, recommended by ECMWF mfpphy.append(cmor_source.grib_code(134)) # Always add the logarithm of surface pressure, recommended by ECMWF mfp2df.append(cmor_source.grib_code(152)) nfp2dfsp, nfp2dfgp = count_spectral_codes(mfp2df) mfp2df = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp2df)))) nfpphysp, nfpphygp = count_spectral_codes(mfpphy) mfpphy = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfpphy)))) nfp3dfssp, nfp3dfsgp = count_spectral_codes(mfp3dfs) mfp3dfs = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp3dfs)))) nfp3dfpsp, nfp3dfpgp = count_spectral_codes(mfp3dfp) mfp3dfp = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp3dfp)))) nfp3dfhsp, nfp3dfhgp = count_spectral_codes(mfp3dfh) mfp3dfh = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp3dfh)))) plevs = sorted(list(set([float(s) for s in plevs])))[::-1] hlevs = sorted(list(set([float(s) for s in hlevs]))) namelist = {"CFPFMT": "MODEL"} if any(mfp2df): namelist["NFP2DF"] = len(mfp2df) namelist["MFP2DF"] = mfp2df num_slices_sp += nfp2dfsp num_slices_gp += nfp2dfgp if any(mfpphy): namelist["NFPPHY"] = len(mfpphy) namelist["MFPPHY"] = mfpphy num_slices_sp += nfpphysp num_slices_gp += nfpphygp if any(mfp3dfs): namelist["NFP3DFS"] = len(mfp3dfs) namelist["MFP3DFS"] = mfp3dfs # To include all model levels use magic number -99. Opposite, by using the magic number -1 the variable is not saved at any model level: namelist["NRFP3S"] = -1 num_blocks_sp += nfp3dfssp num_blocks_gp += nfp3dfsgp if any(mfp3dfp): namelist["NFP3DFP"] = len(mfp3dfp) namelist["MFP3DFP"] = mfp3dfp namelist["RFP3P"] = plevs num_slices_sp += (nfp3dfpsp * len(plevs)) num_slices_gp += (nfp3dfpgp * len(plevs)) if any(mfp3dfh): namelist["NFP3DFH"] = len(mfp3dfh) namelist["MFP3DFH"] = mfp3dfh namelist["RFP3H"] = hlevs num_slices_sp += (nfp3dfhsp * len(hlevs)) num_slices_gp += (nfp3dfhgp * len(hlevs)) num_slices_tot_sp = num_slices_sp if prev_freq == 0 else \ (num_slices_sp + ((freq/prev_freq) - 1) * num_slices_tot_sp) num_slices_tot_gp = num_slices_gp if prev_freq == 0 else \ (num_slices_gp + ((freq/prev_freq) - 1) * num_slices_tot_gp) num_blocks_tot_sp = num_blocks_sp if prev_freq == 0 else \ (num_blocks_sp + ((freq/prev_freq) - 1) * num_blocks_tot_sp) num_blocks_tot_gp = num_blocks_gp if prev_freq == 0 else \ (num_blocks_gp + ((freq/prev_freq) - 1) * num_blocks_tot_gp) prev_freq = freq nml = f90nml.Namelist({"NAMFPC": namelist}) nml.uppercase, nml.end_comma = True, True if freq > 0: f90nml.write(nml, "pptdddddd%04d" % (100 * freq,)) if freq == 0: fx_namelist = namelist if freq == min_freq: # Always add orography and land mask for lowest frequency ppt mfpphy.extend([129, 172, 43]) mfpphy = sorted(list(set(mfpphy))) namelist["MFPPHY"] = mfpphy namelist["NFPPHY"] = len(mfpphy) nml = f90nml.Namelist({"NAMFPC": join_namelists(namelist, fx_namelist)}) nml.uppercase, nml.end_comma = True, True # Write initial state ppt f90nml.write(nml, "ppt0000000000") average_hours_per_month = 730 slices_per_month_sp = (average_hours_per_month * num_slices_tot_sp) / prev_freq slices_per_month_gp = (average_hours_per_month * num_slices_tot_gp) / prev_freq blocks_per_month_sp = (average_hours_per_month * num_blocks_tot_sp) / prev_freq blocks_per_month_gp = (average_hours_per_month * num_blocks_tot_gp) / prev_freq num_layers = 91 log.info("") log.info("EC-Earth IFS output volume estimates:") log.info("---------------------------------------------------------------------------") log.info("# spectral GRIB messages p/m: %d" % (slices_per_month_sp + num_layers * blocks_per_month_sp)) log.info("# gridpoint GRIB messages p/m: %d" % (slices_per_month_gp + num_layers * blocks_per_month_gp)) log.info("---------------------------------------------------------------------------") log.info(" T255L91 T511L91 ") log.info("---------------------------------------------------------------------------") vol255 = (slices_per_month_sp + num_layers * blocks_per_month_sp) * 0.133 / 1000. +\ (slices_per_month_gp + num_layers * blocks_per_month_gp) * 0.180 / 1000. vol511 = (slices_per_month_sp + num_layers * blocks_per_month_sp) * 0.503 / 1000. +\ (slices_per_month_gp + num_layers * blocks_per_month_gp) * 0.698 / 1000. log.info(" %.2f GB/yr %.2f GB/yr " % (12*vol255, 12*vol511)) #volume_estimate = open('volume-estimate-ifs.txt','w') #volume_estimate.write(' \nEC-Earth3 IFS volume estimates of generated output:{}'.format('\n')) #volume_estimate.write(' Volume estimate of the spectral + gridpoint GRIB files for T255L91 grid: {} GB/yr{}'.format(12*vol255, '\n')) #volume_estimate.write(' Volume estimate of the spectral + gridpoint GRIB files for T511L91 grid: {} GB/yr{}'.format(12*vol511, '\n\n')) #volume_estimate.write(' Number of spectral GRIB messages per month: {}{}'.format(slices_per_month_sp + num_layers * blocks_per_month_sp, '\n')) #volume_estimate.write(' Number of gridpoint GRIB messages per month: {}{}'.format(slices_per_month_gp + num_layers * blocks_per_month_gp, '\n\n')) #volume_estimate.close() hf = 3.0 # IFS heuristic factor volume_estimate = open('volume-estimate-ifs.txt','w') volume_estimate.write('{}'.format('\n\n\n')) volume_estimate.write('Heuristic volume estimate for the raw EC-Earth3 IFS output on the T255L91 grid: {:6} GB per year{}'.format(round((12*vol255) / hf, 1), '\n')) volume_estimate.write('Heuristic volume estimate for the raw EC-Earth3 IFS output on the T511L91 grid: {:6} GB per year{}'.format(round((12*vol511) / hf, 1), '\n')) volume_estimate.close()