def get_levels(task, code):
global log
if (code.var_id, code.tab_id) == (134, 128):
return grib_file.surface_level_code, [0]
if 34 < code.var_id < 43 and code.tab_id == 128:
return grib_file.depth_level_code, [0]
if code.var_id in [139, 170, 183, 236] and code.tab_id == 128:
return grib_file.depth_level_code, [0]
zaxis, levels = cmor_target.get_z_axis(task.target)
if zaxis is None:
return grib_file.surface_level_code, [0]
if zaxis in ["sdepth"]:
return grib_file.depth_level_code, [0]
if zaxis in ["alevel", "alevhalf"]:
return grib_file.hybrid_level_code, [-1]
if zaxis == "air_pressure":
return grib_file.pressure_level_Pa_code, [
int(float(l)) for l in levels
]
if zaxis in ["height", "altitude"]:
return grib_file.height_level_code, [int(
float(l)) for l in levels] # TODO: What about decimal places?
log.error(
"Could not convert vertical axis type %s to grib vertical coordinate "
"code for %s" % (zaxis, task.target.variable))
return -1, []
def get_levels(task, code):
global log
# Special cases
if code.tab_id == 128:
gc = code.var_id
if gc in [9, 134]:
return grib_file.surface_level_code, [0]
if gc in [35, 36, 37, 38, 39, 40, 41, 42, 139, 170, 183, 236]:
return grib_file.depth_level_code, [0]
if gc in [49, 165, 166]:
return grib_file.height_level_code, [10]
if gc in [167, 168, 201, 202]:
return grib_file.height_level_code, [2]
# Normal cases
zaxis, levels = cmor_target.get_z_axis(task.target)
if zaxis is None:
return grib_file.surface_level_code, [0]
if zaxis in ["sdepth"]:
return grib_file.depth_level_code, [0]
if zaxis in ["alevel", "alevhalf"]:
return grib_file.hybrid_level_code, [-1]
if zaxis == "air_pressure":
return grib_file.pressure_level_Pa_code, [
int(float(level)) for level in levels
]
if zaxis in ["height", "altitude"]:
return grib_file.height_level_code, [
int(float(level)) for level in levels
] # TODO: What about decimal places?
log.error(
"Could not convert vertical axis type %s to grib vertical coordinate "
"code for %s" % (zaxis, task.target.variable))
return -1, []
def test_validate_tasks():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt1 = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src1 = cmor_source.ifs_source.read("79.128")
tsk1 = cmor_task.cmor_task(src1, tgt1)
tgt2 = ece2cmorlib.get_cmor_target("ua", "Amon")
src2 = cmor_source.ifs_source.read("131.128")
tsk2 = cmor_task.cmor_task(src2, tgt2)
valid_tasks, varstasks = grib_filter.validate_tasks([tsk1, tsk2])
assert valid_tasks == [tsk1, tsk2]
key1 = (79, 128, grib_file.surface_level_code, 0,
cmor_source.ifs_grid.point)
key2 = (131, 128, grib_file.pressure_level_Pa_code, 92500,
cmor_source.ifs_grid.spec)
assert varstasks[key1] == [tsk1]
assert varstasks[key2] == [tsk2]
ltype, plevs = cmor_target.get_z_axis(tgt2)
levs = sorted([float(p) for p in plevs])
levcheck = sorted([k[3] for k in varstasks if k[0] == 131])
assert levs == levcheck
def test_zhalfo_zdims():
abspath = get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
zhalfo = [t for t in targets if t.variable == "zhalfo"][0]
assert cmor_target.get_z_axis(zhalfo)[0] == "olevhalf"
def write_ppt_files(tasks):
freqgroups = cmor_utils.group(tasks, get_output_freq)
for freq1 in freqgroups:
for freq2 in freqgroups:
if freq2 > freq1 and freq2 % freq1 == 0:
freqgroups[freq2] = freqgroups[freq1] + freqgroups[freq2]
for freq in freqgroups:
mfp2df, mfpphy, mfp3dfs, mfp3dfp, mfp3dfv = [], [], [], [], []
alevs, plevs, hlevs = [], [], []
for task in freqgroups[freq]:
zaxis, levs = cmor_target.get_z_axis(task.target)
root_codes = task.source.get_root_codes()
if not zaxis:
for code in root_codes:
if code in cmor_source.ifs_source.grib_codes_3D:
log.warning(
"3D grib code %s used in 2D cmor-target %s..."
"assuming this is on model levels" %
(str(code), task.target.variable))
mfp3dfs.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_dyn:
log.info(
"Adding grib code %s to MFP2DF %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfp2df.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_phy:
log.info(
"Adding grib code %s to MFPPHY %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfpphy.append(code)
else:
log.error("Unknown IFS grib code %s skipped" %
str(code))
else:
for code in root_codes:
if code in cmor_source.ifs_source.grib_codes_3D:
if zaxis in cmor_target.model_axes:
log.info(
"Adding grib code %s to MFP3DFS %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfp3dfs.append(code)
alevs.extend(levs)
elif zaxis in cmor_target.pressure_axes:
log.info(
"Adding grib code %s to MFP3DFP %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfp3dfp.append(code)
plevs.extend(levs)
elif zaxis in cmor_target.height_axes:
log.info(
"Adding grib code %s to MFP3DFV %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfp3dfv.append(code)
hlevs.extend(levs)
else:
log.error(
"Axis type %s unknown, adding grib code %s"
"to model level variables" %
(zaxis, str(code)))
elif code in cmor_source.ifs_source.grib_codes_2D_dyn:
mfp2df.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_phy:
mfpphy.append(code)
else:
log.error("Unknown IFS grib code %s skipped" %
str(code))
# Always add the geopotential, recommended by ECMWF
if cmor_source.grib_code(129) not in mfp3dfs:
mfp2df.append(cmor_source.grib_code(129))
# Always add the surface pressure, recommended by ECMWF
mfpphy.append(cmor_source.grib_code(134))
# Always add the logarithm of surface pressure, recommended by ECMWF
mfp2df.append(cmor_source.grib_code(152))
mfp2df = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfp2df))))
mfpphy = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfpphy))))
mfp3dfs = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfp3dfs))))
mfp3dfp = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfp3dfp))))
mfp3dfv = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfp3dfv))))
plevs = sorted(list(set([float(s) for s in plevs])))[::-1]
hlevs = sorted(list(set([float(s) for s in hlevs])))
namelist = {"CFPFMT": "MODEL"}
if any(mfp2df):
namelist["NFP2DF"] = len(mfp2df)
namelist["MFP2DF"] = mfp2df
if any(mfpphy):
namelist["NFPPHY"] = len(mfpphy)
namelist["MFPPHY"] = mfpphy
if any(mfp3dfs):
namelist["NFP3DFS"] = len(mfp3dfs)
namelist["MFP3DFS"] = mfp3dfs
namelist["NRFP3S"] = -1
if any(mfp3dfp):
namelist["NFP3DFP"] = len(mfp3dfp)
namelist["MFP3DFP"] = mfp3dfp
namelist["RFP3P"] = plevs
if any(mfp3dfs):
namelist["NFP3DFV"] = len(mfp3dfv)
namelist["MFP3DFV"] = mfp3dfv
namelist["RFP3V"] = hlevs
nml = f90nml.Namelist({"NAMFPC": namelist})
nml.uppercase, nml.end_comma = True, True
f90nml.write(nml, "pptdddddd%04d" % (100 * freq, ))
def get_sample_freq(task):
axis, levs = cmor_target.get_z_axis(task.target)
if axis in cmor_target.model_axes + cmor_target.pressure_axes + cmor_target.height_axes:
return 6
else:
return 3
def ifs_model_level_variable(target):
zaxis, levs = cmor_target.get_z_axis(target)
return zaxis not in ["alevel", "alevhalf"]
def write_ppt_files(tasks):
freqgroups = cmor_utils.group(tasks, get_output_freq)
# Fix for issue 313, make sure to always generate 6-hourly ppt:
if freqgroups.keys() == [3]:
freqgroups[6] = []
if -1 in freqgroups.keys():
freqgroups.pop(-1)
freqs_to_remove = []
for freq1 in freqgroups:
if freq1 <= 0:
continue
for freq2 in freqgroups:
if freq2 > freq1:
if freq2 % freq1 == 0:
freqgroups[freq2] = freqgroups[freq1] + freqgroups[freq2]
else:
log.error("Frequency %d is not a divisor of frequency %d: this is not supported, "
"removing the former" % (freq1, freq2))
freqs_to_remove.append(freq1)
for freq in set(freqs_to_remove):
freqgroups.pop(freq, None)
num_slices_tot_sp, num_slices_tot_gp, num_blocks_tot_sp, num_blocks_tot_gp = 0, 0, 0, 0
min_freq = max(freqgroups.keys())
prev_freq = 0
fx_namelist = {}
for freq in sorted(freqgroups.keys()):
mfp2df, mfpphy, mfp3dfs, mfp3dfp, mfp3dfh = [], [], [], [], []
num_slices_sp, num_slices_gp, num_blocks_sp, num_blocks_gp = 0, 0, 0, 0
alevs, plevs, hlevs = [], [], []
for task in freqgroups[freq]:
zaxis, levs = cmor_target.get_z_axis(task.target)
root_codes = task.source.get_root_codes()
if not zaxis:
for code in root_codes:
if freq > 0 and code in cmor_source.ifs_source.grib_codes_fx:
continue
if code in cmor_source.ifs_source.grib_codes_3D:
# Exception for orog and areacella, depend only on lowest level of 129:
if task.target.variable in ["orog", "areacella"] and code == cmor_source.grib_code(129):
mfp2df.append(code)
else:
log.warning("3D grib code %s used in 2D cmor-target %s..."
"assuming this is on model levels" % (str(code), task.target.variable))
mfp3dfs.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_dyn:
log.info("Adding grib code %s to MFP2DF %dhr ppt file for variable "
"%s in table %s" % (str(code), freq, task.target.variable, task.target.table))
mfp2df.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_phy:
log.info("Adding grib code %s to MFPPHY %dhr ppt file for variable "
"%s in table %s" % (str(code), freq, task.target.variable, task.target.table))
mfpphy.append(code)
else:
log.error("Unknown 2D IFS grib code %s skipped" % str(code))
else:
for code in root_codes:
if freq > 0 and code in cmor_source.ifs_source.grib_codes_fx:
continue
if code in cmor_source.ifs_source.grib_codes_3D:
if zaxis in cmor_target.model_axes:
log.info("Adding grib code %s to MFP3DFS %dhr ppt file for variable "
"%s in table %s" % (str(code), freq, task.target.variable, task.target.table))
mfp3dfs.append(code)
alevs.extend(levs)
elif zaxis in cmor_target.pressure_axes:
log.info("Adding grib code %s to MFP3DFP %dhr ppt file for variable "
"%s in table %s" % (str(code), freq, task.target.variable, task.target.table))
mfp3dfp.append(code)
plevs.extend(levs)
elif zaxis in cmor_target.height_axes:
log.info("Adding grib code %s to MFP3DFH %dhr ppt file for variable "
"%s in table %s" % (str(code), freq, task.target.variable, task.target.table))
mfp3dfh.append(code)
hlevs.extend(levs)
else:
log.error("Axis type %s unknown, adding grib code %s"
"to model level variables" % (zaxis, str(code)))
elif code in cmor_source.ifs_source.grib_codes_2D_dyn:
mfp2df.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_phy:
mfpphy.append(code)
# case for PEXTRA tendencies is missing
else:
log.error("Unknown 3D IFS grib code %s skipped" % str(code))
# Always add the geopotential, recommended by ECMWF
if cmor_source.grib_code(129) not in mfp3dfs:
mfp2df.append(cmor_source.grib_code(129))
# Always add the surface pressure, recommended by ECMWF
mfpphy.append(cmor_source.grib_code(134))
# Always add the logarithm of surface pressure, recommended by ECMWF
mfp2df.append(cmor_source.grib_code(152))
nfp2dfsp, nfp2dfgp = count_spectral_codes(mfp2df)
mfp2df = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp2df))))
nfpphysp, nfpphygp = count_spectral_codes(mfpphy)
mfpphy = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfpphy))))
nfp3dfssp, nfp3dfsgp = count_spectral_codes(mfp3dfs)
mfp3dfs = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp3dfs))))
nfp3dfpsp, nfp3dfpgp = count_spectral_codes(mfp3dfp)
mfp3dfp = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp3dfp))))
nfp3dfhsp, nfp3dfhgp = count_spectral_codes(mfp3dfh)
mfp3dfh = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp3dfh))))
plevs = sorted(list(set([float(s) for s in plevs])))[::-1]
hlevs = sorted(list(set([float(s) for s in hlevs])))
namelist = {"CFPFMT": "MODEL"}
if any(mfp2df):
namelist["NFP2DF"] = len(mfp2df)
namelist["MFP2DF"] = mfp2df
num_slices_sp += nfp2dfsp
num_slices_gp += nfp2dfgp
if any(mfpphy):
namelist["NFPPHY"] = len(mfpphy)
namelist["MFPPHY"] = mfpphy
num_slices_sp += nfpphysp
num_slices_gp += nfpphygp
if any(mfp3dfs):
namelist["NFP3DFS"] = len(mfp3dfs)
namelist["MFP3DFS"] = mfp3dfs
# To include all model levels use magic number -99. Opposite, by using the magic number -1 the variable is not saved at any model level:
namelist["NRFP3S"] = -1
num_blocks_sp += nfp3dfssp
num_blocks_gp += nfp3dfsgp
if any(mfp3dfp):
namelist["NFP3DFP"] = len(mfp3dfp)
namelist["MFP3DFP"] = mfp3dfp
namelist["RFP3P"] = plevs
num_slices_sp += (nfp3dfpsp * len(plevs))
num_slices_gp += (nfp3dfpgp * len(plevs))
if any(mfp3dfh):
namelist["NFP3DFH"] = len(mfp3dfh)
namelist["MFP3DFH"] = mfp3dfh
namelist["RFP3H"] = hlevs
num_slices_sp += (nfp3dfhsp * len(hlevs))
num_slices_gp += (nfp3dfhgp * len(hlevs))
num_slices_tot_sp = num_slices_sp if prev_freq == 0 else \
(num_slices_sp + ((freq/prev_freq) - 1) * num_slices_tot_sp)
num_slices_tot_gp = num_slices_gp if prev_freq == 0 else \
(num_slices_gp + ((freq/prev_freq) - 1) * num_slices_tot_gp)
num_blocks_tot_sp = num_blocks_sp if prev_freq == 0 else \
(num_blocks_sp + ((freq/prev_freq) - 1) * num_blocks_tot_sp)
num_blocks_tot_gp = num_blocks_gp if prev_freq == 0 else \
(num_blocks_gp + ((freq/prev_freq) - 1) * num_blocks_tot_gp)
prev_freq = freq
nml = f90nml.Namelist({"NAMFPC": namelist})
nml.uppercase, nml.end_comma = True, True
if freq > 0:
f90nml.write(nml, "pptdddddd%04d" % (100 * freq,))
if freq == 0:
fx_namelist = namelist
if freq == min_freq:
# Always add orography and land mask for lowest frequency ppt
mfpphy.extend([129, 172, 43])
mfpphy = sorted(list(set(mfpphy)))
namelist["MFPPHY"] = mfpphy
namelist["NFPPHY"] = len(mfpphy)
nml = f90nml.Namelist({"NAMFPC": join_namelists(namelist, fx_namelist)})
nml.uppercase, nml.end_comma = True, True
# Write initial state ppt
f90nml.write(nml, "ppt0000000000")
average_hours_per_month = 730
slices_per_month_sp = (average_hours_per_month * num_slices_tot_sp) / prev_freq
slices_per_month_gp = (average_hours_per_month * num_slices_tot_gp) / prev_freq
blocks_per_month_sp = (average_hours_per_month * num_blocks_tot_sp) / prev_freq
blocks_per_month_gp = (average_hours_per_month * num_blocks_tot_gp) / prev_freq
num_layers = 91
log.info("")
log.info("EC-Earth IFS output volume estimates:")
log.info("---------------------------------------------------------------------------")
log.info("# spectral GRIB messages p/m: %d" % (slices_per_month_sp + num_layers * blocks_per_month_sp))
log.info("# gridpoint GRIB messages p/m: %d" % (slices_per_month_gp + num_layers * blocks_per_month_gp))
log.info("---------------------------------------------------------------------------")
log.info(" T255L91 T511L91 ")
log.info("---------------------------------------------------------------------------")
vol255 = (slices_per_month_sp + num_layers * blocks_per_month_sp) * 0.133 / 1000. +\
(slices_per_month_gp + num_layers * blocks_per_month_gp) * 0.180 / 1000.
vol511 = (slices_per_month_sp + num_layers * blocks_per_month_sp) * 0.503 / 1000. +\
(slices_per_month_gp + num_layers * blocks_per_month_gp) * 0.698 / 1000.
log.info(" %.2f GB/yr %.2f GB/yr " % (12*vol255, 12*vol511))
#volume_estimate = open('volume-estimate-ifs.txt','w')
#volume_estimate.write(' \nEC-Earth3 IFS volume estimates of generated output:{}'.format('\n'))
#volume_estimate.write(' Volume estimate of the spectral + gridpoint GRIB files for T255L91 grid: {} GB/yr{}'.format(12*vol255, '\n'))
#volume_estimate.write(' Volume estimate of the spectral + gridpoint GRIB files for T511L91 grid: {} GB/yr{}'.format(12*vol511, '\n\n'))
#volume_estimate.write(' Number of spectral GRIB messages per month: {}{}'.format(slices_per_month_sp + num_layers * blocks_per_month_sp, '\n'))
#volume_estimate.write(' Number of gridpoint GRIB messages per month: {}{}'.format(slices_per_month_gp + num_layers * blocks_per_month_gp, '\n\n'))
#volume_estimate.close()
hf = 3.0 # IFS heuristic factor
volume_estimate = open('volume-estimate-ifs.txt','w')
volume_estimate.write('{}'.format('\n\n\n'))
volume_estimate.write('Heuristic volume estimate for the raw EC-Earth3 IFS output on the T255L91 grid: {:6} GB per year{}'.format(round((12*vol255) / hf, 1), '\n'))
volume_estimate.write('Heuristic volume estimate for the raw EC-Earth3 IFS output on the T511L91 grid: {:6} GB per year{}'.format(round((12*vol511) / hf, 1), '\n'))
volume_estimate.close()