def write_varlist_ascii(targets, opath):
tgtgroups = cmor_utils.group(targets, lambda t: t.table)
ofile = open(opath, 'w')
ofile.write('{:10} {:20} {:5} {:40} {:95} {:20} {:60} {:20} {} {}'.format(
'table', 'variable', 'prio', 'dimensions', 'long_name', 'unit',
'list of MIPs which request this variable', 'comment_author',
'comment', '\n'))
for k, vlist in tgtgroups.iteritems():
ofile.write('{}'.format('\n'))
for tgtvar in vlist:
ofile.write(
'{:10} {:20} {:5} {:40} {:95} {:20} {:60} {:20} {} {}'.format(
tgtvar.table, tgtvar.variable, tgtvar.priority,
getattr(tgtvar, "dimensions", "unknown"),
getattr(tgtvar, "long_name", "unknown"), tgtvar.units,
tgtvar.mip_list, getattr(tgtvar, "comment_author", ""),
getattr(tgtvar, "ecearth_comment", ""), '\n'))
ofile.close()
def write_varlist_ascii(targets, opath, print_all_columns):
tgtgroups = cmor_utils.group(targets, lambda t: t.table)
ofile = open(opath, 'w')
if print_all_columns:
# In case the input data request is an xlsx file, all columns are printed:
ofile.write(
'{:10} {:20} {:5} {:45} {:115} {:20} {:85} {:140} {:20} {} {}'.
format('table', 'variable', 'prio', 'dimensions', 'long_name',
'unit', 'link', 'list of MIPs which request this variable',
'comment_author', 'comment', '\n'))
for k, vlist in tgtgroups.iteritems():
ofile.write('{}'.format('\n'))
for tgtvar in vlist:
ofile.write(
'{:10} {:20} {:5} {:45} {:115} {:20} {:85} {:140} {:20} {} {}'
.format(
tgtvar.table, tgtvar.variable,
getattr(tgtvar, "priority", "unknown"),
getattr(tgtvar, "dimensions", "unknown"),
getattr(tgtvar, "long_name", "unknown"), tgtvar.units,
'http://clipc-services.ceda.ac.uk/dreq/u/' +
getattr(tgtvar, "vid", "unknown") + '.html',
getattr(tgtvar, "mip_list", "unknown"),
getattr(tgtvar, "comment_author", ""),
getattr(tgtvar, "ecearth_comment", ""), '\n'))
else:
# In case the input data request is a json file, a reduced number of columns is printed:
ofile.write('{:10} {:20} {:45} {:115} {:20} {} {}'.format(
'table', 'variable', 'dimensions', 'long_name', 'unit', 'comment',
'\n'))
for k, vlist in tgtgroups.iteritems():
ofile.write('{}'.format('\n'))
for tgtvar in vlist:
ofile.write('{:10} {:20} {:45} {:115} {:20} {} {}'.format(
tgtvar.table, tgtvar.variable,
getattr(tgtvar, "dimensions", "unknown"),
getattr(tgtvar, "long_name", "unknown"), tgtvar.units,
getattr(tgtvar, "ecearth_comment", ""), '\n'))
ofile.close()
logging.info(" Writing the ascii file: %s" % opath)
def cmorize(tasks):
global log, table_root_
log.info("Cmorizing %d IFS tasks..." % len(tasks))
if not any(tasks):
return
cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
cmor.load_table(table_root_ + "_grids.json")
grid = create_grid_from_grib(getattr(tasks[0], "path"))
for task in tasks:
tgtdims = getattr(task.target, cmor_target.dims_key).split()
if "latitude" in tgtdims and "longitude" in tgtdims:
setattr(task, "grid_id", grid)
task_dict = cmor_utils.group(tasks, lambda tsk: tsk.target.table)
for k, v in task_dict.iteritems():
tab = k
task_group = v
log.info("Loading CMOR table %s..." % tab)
try:
tab_id = cmor.load_table("_".join([table_root_, tab]) + ".json")
cmor.set_table(tab_id)
except Exception as e:
log.error(
"CMOR failed to load table %s, the following variables will be skipped: %s. Reason: %s"
% (tab, str([t.target.variable
for t in task_group]), e.message))
continue
log.info("Creating time axes for table %s..." % tab)
create_time_axes(task_group)
log.info("Creating depth axes for table %s..." % tab)
create_depth_axes(task_group)
q = Queue.Queue()
for i in range(postproc.task_threads):
worker = threading.Thread(target=cmor_worker, args=[q])
worker.setDaemon(True)
worker.start()
for task in task_group:
q.put(task)
q.join()
def get_sp_tasks(tasks):
tasks_by_freq = cmor_utils.group(tasks, lambda task: (task.target.frequency,
'_'.join(getattr(task.target, "time_operator", ["mean"]))))
existing, new = [], []
for freq, task_group in tasks_by_freq.iteritems():
tasks3d = [t for t in task_group if "alevel" in getattr(t.target, cmor_target.dims_key).split()]
if not any(tasks3d):
continue
surf_pressure_tasks = [t for t in task_group if t.source.get_grib_code() == surface_pressure]
existing.extend(surf_pressure_tasks)
if any(surf_pressure_tasks):
surf_pressure_task = surf_pressure_tasks[0]
else:
source = cmor_source.ifs_source(surface_pressure)
surf_pressure_task = cmor_task.cmor_task(source, cmor_target.cmor_target("ps", tasks3d[0].target.table))
setattr(surf_pressure_task.target, cmor_target.freq_key, freq[0])
setattr(surf_pressure_task.target, "time_operator", freq[1].split('_'))
setattr(surf_pressure_task.target, cmor_target.dims_key, "latitude longitude")
find_sp_variable(surf_pressure_task)
new.append(surf_pressure_task)
for task3d in tasks3d:
setattr(task3d, "sp_task", surf_pressure_task)
return existing, new
def write_ppt_files(tasks):
freqgroups = cmor_utils.group(tasks, get_output_freq)
for freq1 in freqgroups:
for freq2 in freqgroups:
if freq2 > freq1 and freq2 % freq1 == 0:
freqgroups[freq2] = freqgroups[freq1] + freqgroups[freq2]
for freq in freqgroups:
mfp2df, mfpphy, mfp3dfs, mfp3dfp, mfp3dfv = [], [], [], [], []
alevs, plevs, hlevs = [], [], []
for task in freqgroups[freq]:
zaxis, levs = cmor_target.get_z_axis(task.target)
root_codes = task.source.get_root_codes()
if not zaxis:
for code in root_codes:
if code in cmor_source.ifs_source.grib_codes_3D:
log.warning(
"3D grib code %s used in 2D cmor-target %s..."
"assuming this is on model levels" %
(str(code), task.target.variable))
mfp3dfs.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_dyn:
log.info(
"Adding grib code %s to MFP2DF %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfp2df.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_phy:
log.info(
"Adding grib code %s to MFPPHY %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfpphy.append(code)
else:
log.error("Unknown IFS grib code %s skipped" %
str(code))
else:
for code in root_codes:
if code in cmor_source.ifs_source.grib_codes_3D:
if zaxis in cmor_target.model_axes:
log.info(
"Adding grib code %s to MFP3DFS %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfp3dfs.append(code)
alevs.extend(levs)
elif zaxis in cmor_target.pressure_axes:
log.info(
"Adding grib code %s to MFP3DFP %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfp3dfp.append(code)
plevs.extend(levs)
elif zaxis in cmor_target.height_axes:
log.info(
"Adding grib code %s to MFP3DFV %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfp3dfv.append(code)
hlevs.extend(levs)
else:
log.error(
"Axis type %s unknown, adding grib code %s"
"to model level variables" %
(zaxis, str(code)))
elif code in cmor_source.ifs_source.grib_codes_2D_dyn:
mfp2df.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_phy:
mfpphy.append(code)
else:
log.error("Unknown IFS grib code %s skipped" %
str(code))
# Always add the geopotential, recommended by ECMWF
if cmor_source.grib_code(129) not in mfp3dfs:
mfp2df.append(cmor_source.grib_code(129))
# Always add the surface pressure, recommended by ECMWF
mfpphy.append(cmor_source.grib_code(134))
# Always add the logarithm of surface pressure, recommended by ECMWF
mfp2df.append(cmor_source.grib_code(152))
mfp2df = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfp2df))))
mfpphy = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfpphy))))
mfp3dfs = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfp3dfs))))
mfp3dfp = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfp3dfp))))
mfp3dfv = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfp3dfv))))
plevs = sorted(list(set([float(s) for s in plevs])))[::-1]
hlevs = sorted(list(set([float(s) for s in hlevs])))
namelist = {"CFPFMT": "MODEL"}
if any(mfp2df):
namelist["NFP2DF"] = len(mfp2df)
namelist["MFP2DF"] = mfp2df
if any(mfpphy):
namelist["NFPPHY"] = len(mfpphy)
namelist["MFPPHY"] = mfpphy
if any(mfp3dfs):
namelist["NFP3DFS"] = len(mfp3dfs)
namelist["MFP3DFS"] = mfp3dfs
namelist["NRFP3S"] = -1
if any(mfp3dfp):
namelist["NFP3DFP"] = len(mfp3dfp)
namelist["MFP3DFP"] = mfp3dfp
namelist["RFP3P"] = plevs
if any(mfp3dfs):
namelist["NFP3DFV"] = len(mfp3dfv)
namelist["MFP3DFV"] = mfp3dfv
namelist["RFP3V"] = hlevs
nml = f90nml.Namelist({"NAMFPC": namelist})
nml.uppercase, nml.end_comma = True, True
f90nml.write(nml, "pptdddddd%04d" % (100 * freq, ))
def write_varlist_excel(targets, opath, with_pingfile):
import xlsxwriter
tgtgroups = cmor_utils.group(targets, lambda t: t.table)
workbook = xlsxwriter.Workbook(opath)
worksheet = workbook.add_worksheet()
worksheet.set_column('A:A', 10) # Adjust the column width of column A
worksheet.set_column('B:B', 15) # Adjust the column width of column B
worksheet.set_column('C:C', 4) # Adjust the column width of column C
worksheet.set_column('D:D', 35) # Adjust the column width of column D
worksheet.set_column('E:E', 80) # Adjust the column width of column E
worksheet.set_column('F:F', 15) # Adjust the column width of column E
worksheet.set_column('G:G', 4) # Adjust the column width of column F
worksheet.set_column('H:H', 80) # Adjust the column width of column G
worksheet.set_column('I:I', 15) # Adjust the column width of column H
worksheet.set_column('J:J', 200) # Adjust the column width of column I
worksheet.set_column('K:K', 80) # Adjust the column width of column J
if with_pingfile:
worksheet.set_column('L:L', 28) # Adjust the column width of column L
worksheet.set_column('M:M', 17) # Adjust the column width of column M
worksheet.set_column('N:N', 100) # Adjust the column width of column N
bold = workbook.add_format({'bold': True}) # Add a bold format
worksheet.write(0, 0, 'Table', bold)
worksheet.write(0, 1, 'variable', bold)
worksheet.write(0, 2, 'prio', bold)
worksheet.write(0, 3, 'Dimension format of variable', bold)
worksheet.write(0, 4, 'variable long name', bold)
worksheet.write(0, 5, 'unit', bold)
worksheet.write(0, 6, 'link', bold)
worksheet.write(0, 7, 'comment', bold)
worksheet.write(0, 8, 'comment author', bold)
worksheet.write(0, 9, 'extensive variable description', bold)
worksheet.write(0, 10, 'list of MIPs which request this variable', bold)
if with_pingfile:
worksheet.write(0, 11, 'model component in ping file', bold)
worksheet.write(0, 12, 'units as in ping file', bold)
worksheet.write(0, 13, 'ping file comment', bold)
row_counter = 1
for k, vlist in tgtgroups.iteritems():
worksheet.write(row_counter, 0, '')
row_counter += 1
for tgtvar in vlist:
worksheet.write(row_counter, 0, tgtvar.table)
worksheet.write(row_counter, 1, tgtvar.variable)
worksheet.write(row_counter, 2,
getattr(tgtvar, "priority", "unknown"))
worksheet.write(row_counter, 3,
getattr(tgtvar, "dimensions", "unknown"))
worksheet.write(row_counter, 4,
getattr(tgtvar, "long_name", "unknown"))
worksheet.write(row_counter, 5, tgtvar.units)
worksheet.write(
row_counter, 6,
'=HYPERLINK("' + 'http://clipc-services.ceda.ac.uk/dreq/u/' +
getattr(tgtvar, "vid", "unknown") + '.html","web")')
worksheet.write(row_counter, 7,
getattr(tgtvar, "ecearth_comment", ""))
worksheet.write(row_counter, 8,
getattr(tgtvar, "comment_author", ""))
worksheet.write(row_counter, 9,
getattr(tgtvar, "comment", "unknown"))
worksheet.write(row_counter, 10,
getattr(tgtvar, "mip_list", "unknown"))
if with_pingfile:
worksheet.write(row_counter, 11, getattr(tgtvar, "model", ""))
worksheet.write(row_counter, 12, getattr(tgtvar, "units", ""))
worksheet.write(row_counter, 13,
getattr(tgtvar, "pingcomment", ""))
row_counter += 1
workbook.close()
logging.info(" Writing the excel file: %s" % opath)
def execute(tasks):
global log, table_root_
global lpjg_path_, ncpath_
log.info("Executing %d lpjg tasks..." % len(tasks))
log.info("Cmorizing lpjg tasks...")
taskdict = cmor_utils.group(tasks, lambda t: t.target.table)
for table, tasklist in taskdict.iteritems():
try:
tab_id = cmor.load_table("_".join([table_root_, table]) + ".json")
cmor.set_table(tab_id)
except Exception as e:
log.error("CMOR failed to load table %s, skipping variables %s. Reason: %s"
% (table, ','.join([tsk.target.variable for tsk in tasklist]), e.message))
continue
lon_id = None
lat_id = None
for task in tasklist:
freq = task.target.frequency.encode()
freqstr = get_lpj_freq(task.target.frequency)
if freqstr is None:
log.error("The frequency %s for variable %s in table %s is not supported by lpj2cmor" %
(task.target.frequency, task.target.variable, task.target.table))
task.set_failed()
continue
# if compressed file .out.gz file exists, uncompress it to temporary .out file
gzfile = os.path.join(lpjg_path_, task.source.variable() + "_" + freqstr + ".out.gz")
if os.path.exists(gzfile):
lpjgfile = os.path.join(ncpath_, task.source.variable() + "_" + freqstr + ".out")
log.info("Uncompressing file "+gzfile+" to temporary file "+lpjgfile)
with gzip.open(gzfile, 'rb') as f_in, open(lpjgfile, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
else:
lpjgfile = os.path.join(lpjg_path_, task.source.variable() + "_" + freqstr + ".out")
if not os.path.exists(lpjgfile):
log.error("The file %s does not exist. Skipping CMORization of variable %s."
% (lpjgfile, task.source.variable()))
task.set_failed()
continue
if not check_time_resolution(lpjgfile, freq):
log.error("The data in the file %s did not match the expected frequency (time resolution). "
"Skipping CMORization of variable %s." % (lpjgfile, task.source.variable()))
task.set_failed()
continue
log.info("Processing file "+lpjgfile)
setattr(task, cmor_task.output_path_key, task.source.variable() + ".out")
outname = task.target.out_name
outdims = task.target.dimensions
# find first and last year in the .out-file
firstyear, lastyear = find_timespan(lpjgfile)
# check if user given reference year is after the first year in data file: this is not allowed
if int(ref_date_.year) > firstyear:
log.error("The reference date given is after the first year in the data (%s) for variable %s "
"in file %s. Skipping CMORization." % (firstyear, task.source.variable(), lpjgfile))
task.set_failed()
continue
# divide the data in the .out-file by year to temporary files
yearly_files = divide_years(lpjgfile, firstyear, lastyear, outname)
for yearfile in yearly_files:
# Read data from the .out-file and generate the netCDF file including remapping
ncfile = create_lpjg_netcdf(freq, yearfile, outname, outdims)
if ncfile is None:
if "landUse" in outdims.split():
log.error("Land use columns in file %s do not contain all of the requested land use types. "
"Skipping CMORization of variable %s" % (
getattr(task, cmor_task.output_path_key), task.source.variable()))
else:
log.error(
"Unexpected subtype in file %s: either no type axis has been requested for variable %s "
"or explicit treatment for the axis has not yet been implemented. Skipping CMORization."
% (getattr(task, cmor_task.output_path_key), task.source.variable()))
break
dataset = netCDF4.Dataset(ncfile, 'r')
# Create the grid, need to do only once as all LPJG variables will be on same grid
# Currently create_grid just creates latitude and longitude axis since that should be all that is needed
if lon_id is None and lat_id is None:
lon_id, lat_id = create_grid(dataset, task)
setattr(task, "longitude_axis", lon_id)
setattr(task, "latitude_axis", lat_id)
# Create cmor time axis for current variable
create_time_axis(dataset, task)
# if this is a land use variable create cmor land use axis
if "landUse" in outdims.split():
create_landuse_axis(task, lpjgfile, freq)
# if this is a pft variable (e.g. landCoverFrac) create cmor vegtype axis
if "vegtype" in outdims.split():
create_vegtype_axis(task, lpjgfile, freq)
# if this variable has the soil depth dimension sdepth
# (NB! not sdepth1 or sdepth10) create cmor sdepth axis
if "sdepth" in outdims.split():
create_sdepth_axis(task, lpjgfile, freq)
# if this variable has one or more "singleton axes" (i.e. axes
# of length 1) which can be those dimensions
# named "type*", these will be created here
for lpjgcol in outdims.split():
if lpjgcol.startswith("type"):
# THIS SHOULD BE LINKED TO CIP6_coordinate.json!
if lpjgcol == "typenwd":
singleton_value = "herbaceous_vegetation"
elif lpjgcol == "typepasture":
singleton_value = "pastures"
else:
continue
create_singleton_axis(task, lpjgfile, str(lpjgcol), singleton_value)
# cmorize the current task (variable)
execute_single_task(dataset, task)
dataset.close()
# remove the regular (non-cmorized) netCDF file and the temporary .out file for current year
os.remove(ncfile)
os.remove(yearfile)
# end yearfile loop
# if compressed file .out.gz file exists, remove temporary .out file
if os.path.exists(gzfile):
os.remove(lpjgfile)
return
def write_varlist(targets,opath):
tgtgroups = cmor_utils.group(targets,lambda t:t.table)
tgtdict = dict([k,[t.variable for t in v]] for k,v in tgtgroups.iteritems())
with open(opath,'w') as ofile:
json.dump(tgtdict,ofile,indent = 4,separators = (',', ': '))
def execute(tasks, nthreads=1):
global log, start_date_, auto_filter_
supported_tasks = [t for t in filter_tasks(tasks) if t.status == cmor_task.status_initialized]
log.info("Executing %d IFS tasks..." % len(supported_tasks))
mask_tasks = get_mask_tasks(supported_tasks)
fx_tasks = [t for t in supported_tasks if cmor_target.get_freq(t.target) == 0]
regular_tasks = [t for t in supported_tasks if cmor_target.get_freq(t.target) != 0]
script_tasks = [t for t in supported_tasks if validate_script_task(t) is not None]
# Scripts in charge of their own filtering, can create a group of variables at once
script_tasks_no_filter = [t for t in script_tasks if validate_script_task(t) == "false"]
# Scripts creating single variable, filtering done by ece2cmor3
script_tasks_filter = list(set(script_tasks) - set(script_tasks_no_filter))
req_ps_tasks, extra_ps_tasks = get_sp_tasks(supported_tasks)
# No fx filtering needed, cdo can handle this file
if ifs_init_gridpoint_file_.endswith("+000000"):
tasks_to_filter = extra_ps_tasks + regular_tasks + script_tasks_filter
tasks_no_filter = fx_tasks + mask_tasks
for task in tasks_no_filter:
# dirty hack for orography being in ICMGG+000000 file...
if task.target.variable in ["orog", "areacella"]:
task.source.grid_ = cmor_source.ifs_grid.point
if task.source.grid_id() == cmor_source.ifs_grid.spec:
setattr(task, cmor_task.filter_output_key, [ifs_init_spectral_file_])
else:
setattr(task, cmor_task.filter_output_key, [ifs_init_gridpoint_file_])
setattr(task, cmor_task.output_frequency_key, 0)
else:
tasks_to_filter = mask_tasks + fx_tasks + extra_ps_tasks + regular_tasks + script_tasks_filter
tasks_no_filter = []
np = nthreads
# Launch no-filter scripts
jobs = []
tasks_per_script = cmor_utils.group(script_tasks_no_filter, lambda tsk: getattr(tsk, cmor_task.postproc_script_key))
for s, tasklist in tasks_per_script.items():
log.info("Launching script %s to process variables %s" %
(s, ','.join([t.target.variable + " in " + t.target.table for t in tasklist])))
script_args = (s, str(scripts[s]["src"]), tasklist)
if np == 1:
script_worker(*script_args)
else:
p = multiprocessing.Process(name=s, target=script_worker, args=script_args)
p.start()
jobs.append(p)
np -= 1
# Do filtering
if auto_filter_:
tasks_todo = tasks_no_filter + grib_filter.execute(tasks_to_filter, filter_files=do_post_process(),
multi_threaded=(nthreads > 1))
else:
tasks_todo = tasks_no_filter
for task in tasks_to_filter:
if task.source.grid_id() == cmor_source.ifs_grid.point:
setattr(task, cmor_task.filter_output_key, ifs_gridpoint_files_.values())
tasks_todo.append(task)
elif task.source.grid_id() == cmor_source.ifs_grid.spec:
setattr(task, cmor_task.filter_output_key, ifs_spectral_files_.values())
tasks_todo.append(task)
else:
log.error("Task ifs source has unknown grid for %s in table %s" % (task.target.variable,
task.target.table))
task.set_failed()
for task in tasks_todo:
setattr(task, cmor_task.output_frequency_key, get_output_freq(task))
# First post-process surface pressure and mask tasks
for task in list(set(tasks_todo).intersection(mask_tasks + req_ps_tasks + extra_ps_tasks)):
postproc.post_process(task, temp_dir_, do_post_process())
for task in list(set(tasks_todo).intersection(mask_tasks)):
read_mask(task.target.variable, getattr(task, cmor_task.output_path_key))
proctasks = list(set(tasks_todo).intersection(regular_tasks + fx_tasks))
if np == 1:
for task in proctasks:
cmor_worker(task)
else:
pool = multiprocessing.Pool(processes=np)
pool.map(cmor_worker, proctasks)
for task in proctasks:
setattr(task, cmor_task.output_path_key, postproc.get_output_path(task, temp_dir_))
for job in jobs:
job.join()
if cleanup_tmpdir():
clean_tmp_data(tasks_todo)
def write_ppt_files(tasks):
freqgroups = cmor_utils.group(tasks, get_output_freq)
# Fix for issue 313, make sure to always generate 6-hourly ppt:
if freqgroups.keys() == [3]:
freqgroups[6] = []
if -1 in freqgroups.keys():
freqgroups.pop(-1)
freqs_to_remove = []
for freq1 in freqgroups:
if freq1 <= 0:
continue
for freq2 in freqgroups:
if freq2 > freq1:
if freq2 % freq1 == 0:
freqgroups[freq2] = freqgroups[freq1] + freqgroups[freq2]
else:
log.error("Frequency %d is not a divisor of frequency %d: this is not supported, "
"removing the former" % (freq1, freq2))
freqs_to_remove.append(freq1)
for freq in set(freqs_to_remove):
freqgroups.pop(freq, None)
num_slices_tot_sp, num_slices_tot_gp, num_blocks_tot_sp, num_blocks_tot_gp = 0, 0, 0, 0
min_freq = max(freqgroups.keys())
prev_freq = 0
fx_namelist = {}
for freq in sorted(freqgroups.keys()):
mfp2df, mfpphy, mfp3dfs, mfp3dfp, mfp3dfh = [], [], [], [], []
num_slices_sp, num_slices_gp, num_blocks_sp, num_blocks_gp = 0, 0, 0, 0
alevs, plevs, hlevs = [], [], []
for task in freqgroups[freq]:
zaxis, levs = cmor_target.get_z_axis(task.target)
root_codes = task.source.get_root_codes()
if not zaxis:
for code in root_codes:
if freq > 0 and code in cmor_source.ifs_source.grib_codes_fx:
continue
if code in cmor_source.ifs_source.grib_codes_3D:
# Exception for orog and areacella, depend only on lowest level of 129:
if task.target.variable in ["orog", "areacella"] and code == cmor_source.grib_code(129):
mfp2df.append(code)
else:
log.warning("3D grib code %s used in 2D cmor-target %s..."
"assuming this is on model levels" % (str(code), task.target.variable))
mfp3dfs.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_dyn:
log.info("Adding grib code %s to MFP2DF %dhr ppt file for variable "
"%s in table %s" % (str(code), freq, task.target.variable, task.target.table))
mfp2df.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_phy:
log.info("Adding grib code %s to MFPPHY %dhr ppt file for variable "
"%s in table %s" % (str(code), freq, task.target.variable, task.target.table))
mfpphy.append(code)
else:
log.error("Unknown 2D IFS grib code %s skipped" % str(code))
else:
for code in root_codes:
if freq > 0 and code in cmor_source.ifs_source.grib_codes_fx:
continue
if code in cmor_source.ifs_source.grib_codes_3D:
if zaxis in cmor_target.model_axes:
log.info("Adding grib code %s to MFP3DFS %dhr ppt file for variable "
"%s in table %s" % (str(code), freq, task.target.variable, task.target.table))
mfp3dfs.append(code)
alevs.extend(levs)
elif zaxis in cmor_target.pressure_axes:
log.info("Adding grib code %s to MFP3DFP %dhr ppt file for variable "
"%s in table %s" % (str(code), freq, task.target.variable, task.target.table))
mfp3dfp.append(code)
plevs.extend(levs)
elif zaxis in cmor_target.height_axes:
log.info("Adding grib code %s to MFP3DFH %dhr ppt file for variable "
"%s in table %s" % (str(code), freq, task.target.variable, task.target.table))
mfp3dfh.append(code)
hlevs.extend(levs)
else:
log.error("Axis type %s unknown, adding grib code %s"
"to model level variables" % (zaxis, str(code)))
elif code in cmor_source.ifs_source.grib_codes_2D_dyn:
mfp2df.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_phy:
mfpphy.append(code)
# case for PEXTRA tendencies is missing
else:
log.error("Unknown 3D IFS grib code %s skipped" % str(code))
# Always add the geopotential, recommended by ECMWF
if cmor_source.grib_code(129) not in mfp3dfs:
mfp2df.append(cmor_source.grib_code(129))
# Always add the surface pressure, recommended by ECMWF
mfpphy.append(cmor_source.grib_code(134))
# Always add the logarithm of surface pressure, recommended by ECMWF
mfp2df.append(cmor_source.grib_code(152))
nfp2dfsp, nfp2dfgp = count_spectral_codes(mfp2df)
mfp2df = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp2df))))
nfpphysp, nfpphygp = count_spectral_codes(mfpphy)
mfpphy = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfpphy))))
nfp3dfssp, nfp3dfsgp = count_spectral_codes(mfp3dfs)
mfp3dfs = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp3dfs))))
nfp3dfpsp, nfp3dfpgp = count_spectral_codes(mfp3dfp)
mfp3dfp = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp3dfp))))
nfp3dfhsp, nfp3dfhgp = count_spectral_codes(mfp3dfh)
mfp3dfh = sorted(list(map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(), set(mfp3dfh))))
plevs = sorted(list(set([float(s) for s in plevs])))[::-1]
hlevs = sorted(list(set([float(s) for s in hlevs])))
namelist = {"CFPFMT": "MODEL"}
if any(mfp2df):
namelist["NFP2DF"] = len(mfp2df)
namelist["MFP2DF"] = mfp2df
num_slices_sp += nfp2dfsp
num_slices_gp += nfp2dfgp
if any(mfpphy):
namelist["NFPPHY"] = len(mfpphy)
namelist["MFPPHY"] = mfpphy
num_slices_sp += nfpphysp
num_slices_gp += nfpphygp
if any(mfp3dfs):
namelist["NFP3DFS"] = len(mfp3dfs)
namelist["MFP3DFS"] = mfp3dfs
# To include all model levels use magic number -99. Opposite, by using the magic number -1 the variable is not saved at any model level:
namelist["NRFP3S"] = -1
num_blocks_sp += nfp3dfssp
num_blocks_gp += nfp3dfsgp
if any(mfp3dfp):
namelist["NFP3DFP"] = len(mfp3dfp)
namelist["MFP3DFP"] = mfp3dfp
namelist["RFP3P"] = plevs
num_slices_sp += (nfp3dfpsp * len(plevs))
num_slices_gp += (nfp3dfpgp * len(plevs))
if any(mfp3dfh):
namelist["NFP3DFH"] = len(mfp3dfh)
namelist["MFP3DFH"] = mfp3dfh
namelist["RFP3H"] = hlevs
num_slices_sp += (nfp3dfhsp * len(hlevs))
num_slices_gp += (nfp3dfhgp * len(hlevs))
num_slices_tot_sp = num_slices_sp if prev_freq == 0 else \
(num_slices_sp + ((freq/prev_freq) - 1) * num_slices_tot_sp)
num_slices_tot_gp = num_slices_gp if prev_freq == 0 else \
(num_slices_gp + ((freq/prev_freq) - 1) * num_slices_tot_gp)
num_blocks_tot_sp = num_blocks_sp if prev_freq == 0 else \
(num_blocks_sp + ((freq/prev_freq) - 1) * num_blocks_tot_sp)
num_blocks_tot_gp = num_blocks_gp if prev_freq == 0 else \
(num_blocks_gp + ((freq/prev_freq) - 1) * num_blocks_tot_gp)
prev_freq = freq
nml = f90nml.Namelist({"NAMFPC": namelist})
nml.uppercase, nml.end_comma = True, True
if freq > 0:
f90nml.write(nml, "pptdddddd%04d" % (100 * freq,))
if freq == 0:
fx_namelist = namelist
if freq == min_freq:
# Always add orography and land mask for lowest frequency ppt
mfpphy.extend([129, 172, 43])
mfpphy = sorted(list(set(mfpphy)))
namelist["MFPPHY"] = mfpphy
namelist["NFPPHY"] = len(mfpphy)
nml = f90nml.Namelist({"NAMFPC": join_namelists(namelist, fx_namelist)})
nml.uppercase, nml.end_comma = True, True
# Write initial state ppt
f90nml.write(nml, "ppt0000000000")
average_hours_per_month = 730
slices_per_month_sp = (average_hours_per_month * num_slices_tot_sp) / prev_freq
slices_per_month_gp = (average_hours_per_month * num_slices_tot_gp) / prev_freq
blocks_per_month_sp = (average_hours_per_month * num_blocks_tot_sp) / prev_freq
blocks_per_month_gp = (average_hours_per_month * num_blocks_tot_gp) / prev_freq
num_layers = 91
log.info("")
log.info("EC-Earth IFS output volume estimates:")
log.info("---------------------------------------------------------------------------")
log.info("# spectral GRIB messages p/m: %d" % (slices_per_month_sp + num_layers * blocks_per_month_sp))
log.info("# gridpoint GRIB messages p/m: %d" % (slices_per_month_gp + num_layers * blocks_per_month_gp))
log.info("---------------------------------------------------------------------------")
log.info(" T255L91 T511L91 ")
log.info("---------------------------------------------------------------------------")
vol255 = (slices_per_month_sp + num_layers * blocks_per_month_sp) * 0.133 / 1000. +\
(slices_per_month_gp + num_layers * blocks_per_month_gp) * 0.180 / 1000.
vol511 = (slices_per_month_sp + num_layers * blocks_per_month_sp) * 0.503 / 1000. +\
(slices_per_month_gp + num_layers * blocks_per_month_gp) * 0.698 / 1000.
log.info(" %.2f GB/yr %.2f GB/yr " % (12*vol255, 12*vol511))
#volume_estimate = open('volume-estimate-ifs.txt','w')
#volume_estimate.write(' \nEC-Earth3 IFS volume estimates of generated output:{}'.format('\n'))
#volume_estimate.write(' Volume estimate of the spectral + gridpoint GRIB files for T255L91 grid: {} GB/yr{}'.format(12*vol255, '\n'))
#volume_estimate.write(' Volume estimate of the spectral + gridpoint GRIB files for T511L91 grid: {} GB/yr{}'.format(12*vol511, '\n\n'))
#volume_estimate.write(' Number of spectral GRIB messages per month: {}{}'.format(slices_per_month_sp + num_layers * blocks_per_month_sp, '\n'))
#volume_estimate.write(' Number of gridpoint GRIB messages per month: {}{}'.format(slices_per_month_gp + num_layers * blocks_per_month_gp, '\n\n'))
#volume_estimate.close()
hf = 3.0 # IFS heuristic factor
volume_estimate = open('volume-estimate-ifs.txt','w')
volume_estimate.write('{}'.format('\n\n\n'))
volume_estimate.write('Heuristic volume estimate for the raw EC-Earth3 IFS output on the T255L91 grid: {:6} GB per year{}'.format(round((12*vol255) / hf, 1), '\n'))
volume_estimate.write('Heuristic volume estimate for the raw EC-Earth3 IFS output on the T511L91 grid: {:6} GB per year{}'.format(round((12*vol511) / hf, 1), '\n'))
volume_estimate.close()