def fill_homevar(home_var):
logger = get_logger()
if home_var.spatial_shp in ["XY-perso", "XY-HG"]:
home_var_sdims_info = get_settings_values("internal",
'perso_sdims_description')
if home_var.label in home_var_sdims_info:
home_var_sdims = OrderedDict()
for home_var_dim in home_var_sdims_info[home_var.label]:
home_var_sdim = SimpleDim()
home_var_sdim.label = home_var_dim
for sdim_key in [
"zoom_label", "stdname", "long_name", "positive",
"requested", "value", "out_name", "units",
"is_zoom_of", "bounds", "boundsValue", "axis", "type",
"coords", "title", "is_union_for"
]:
if sdim_key in home_var_sdims_info[
home_var.label][home_var_dim]:
setattr(
home_var_sdim, sdim_key, home_var_sdims_info[
home_var.label][home_var_dim][sdim_key])
home_var_sdims[home_var_dim] = home_var_sdim
home_var.sdims = home_var_sdims
else:
logger.error(
"Could not find custom sdims for {} in simulation settings.".
format(home_var.label))
raise VarsError(
"Could not find custom sdims for %s in simulation settings." %
home_var.label)
actual_mip = home_var.mip
if actual_mip.startswith(r"[") and actual_mip.endswith(r"]"):
home_var.mip = actual_mip.lstrip("[").rstrip("]").split(",")
return home_var
def check_dev_variable(home_var, hv_info):
logger = get_logger()
is_cmor = get_correspond_cmor_var(home_var)
if not is_cmor:
if home_var.mipVarLabel is None:
home_var.set_attributes(mipVarLabel=home_var.label)
if any([cmvar.label == home_var.label for cmvar in get_list_of_elements_by_id("CMORvar").items]):
raise VarsError("Error: %s "
"HOMEVar is announced as dev, is not a CMORVar, but has a cmor name. "
"=> Not taken into account." % hv_info)
else:
logger.debug("Info: %s HOMEVar is purely dev. => Taken into account." % hv_info)
return home_var
else:
raise VarsError("Error: %s "
"HOMEVar is announced as dev, but in reality is cmor => Not taken into account." %
hv_info)
def process_home_vars(mip_vars_list, mips, expid="False"):
"""
Deal with home variables
:param mip_vars_list:
:param mips:
:param expid:
:return:
"""
logger = get_logger()
internal_dict = get_settings_values("internal")
# Read HOME variables
homevars = internal_dict['listof_home_vars']
path_extra_tables = internal_dict['path_extra_tables']
logger.info("homevars file: %s" % homevars)
home_vars_list = read_home_vars_list(homevars, expid, mips,
path_extra_tables)
logger.info("homevars list: %s" %
" ".join([sv.label for sv in home_vars_list]))
#
for hv in home_vars_list:
hv_info = {
"varname": hv.label,
"realm": hv.modeling_realm,
"freq": hv.frequency,
"table": hv.mipTable
}
logger.debug(hv_info)
if hv.type in [
"cmor",
]:
new_hv = check_cmor_variable(hv, mip_vars_list, hv_info)
elif hv.type in [
"perso",
]:
new_hv = check_perso_variable(hv, hv_info)
elif hv.type in [
"dev",
]:
new_hv = check_dev_variable(hv, hv_info)
elif hv.type in [
"extra",
]:
new_hv = check_extra_variable(hv, hv_info)
else:
logger.error("%s HOMEVar type %s")
raise VarsError(
"Error: %s HOMEVar type %s does not correspond to any known keyword. "
"=> Not taken into account." % (hv_info, hv.type))
if new_hv:
mip_vars_list.append(new_hv)
return mip_vars_list
def read_home_var_extra(line_split,
expid,
mips,
path_extra_tables=None,
extra_vars_per_table=defaultdict(list)):
logger = get_logger()
rep = list()
home_var = read_home_var(line_split, home_attrs)
table = home_var.mipTable
if table not in "NONE" and "_" not in table:
logger.error("A prefix is expected in extra Table name %s" % table)
raise VarsError("A prefix is expected in extra Table name %s" % table)
if table not in extra_vars_per_table:
extra_vars_per_table[table] = read_extra_table(path_extra_tables,
table)
if home_var.label in [
"ANY",
]:
var_found = extra_vars_per_table[table]
else:
# find home_var in extra_vars
var_found = [
var for var in extra_vars_per_table[table]
if var.label == home_var.label
]
if len(var_found) == 0:
print("Warning: 'extra' variable %s not found in table %s" %
(home_var.label, table))
var_found = list()
else:
var_found = var_found[0]
if home_var.ref_var is not None:
var_found.ref_var = home_var.ref_var
var_found = [
var_found,
]
if home_var.mip in [
"ANY",
] or home_var.mip in mips:
if home_var.experiment not in [
"ANY",
]:
if expid in home_var.experiment:
rep.extend(var_found)
else:
rep.extend(var_found)
return rep
def read_home_var_dev(line_split, mips, expid):
if len(line_split) < len(home_attrs) + 1:
raise VarsError("Missing geometry description for dev variable {}.".format(line_split[1]))
else:
additional_args = line_split[-2:]
line_split = line_split[:-1]
line_split[-1] = "|".join(additional_args)
home_var = read_home_var(line_split, home_attrs)
if home_var.frequency in ["fx", ]:
cell_methods = None
else:
cell_methods = tcmName2tcmValue[home_var.temporal_shp]
home_var.set_attributes(label_with_area=home_var.label, mip_era="DEV", mipVarLabel=home_var.label,
cell_methods=cell_methods, label_without_psuffix=home_var.label, cell_measures="")
home_var = fill_homevar(home_var)
if check_homevar(home_var, mips, expid):
return home_var
else:
return None
def read_extra_table(path, table):
logger = get_logger()
#
dims2shape, dim2dimid, dr_single_levels = initialize_dim_variables()
#
extravars = list()
dim_from_extra = defaultdict(OrderedDict)
dynamic_shapes = defaultdict(OrderedDict)
#
mip_era, tbl = table.split('_')
json_table = path + "/" + table + ".json"
json_coordinate = path + "/" + mip_era + "_coordinate.json"
if not os.path.exists(json_table):
logger.error("Abort: file for extra Table does not exist: " +
json_table)
raise VarsError("Abort: file for extra Table does not exist: " +
json_table)
else:
tdata = read_json_content(json_table)
for k, v in tdata["variable_entry"].items():
if "frequency" in v:
freq = v["frequency"]
else:
freq = guess_freq_from_table_name(tbl)
extra_var = SimpleCMORVar(type="extra",
mip_era=mip_era,
label=v["out_name"],
mipVarLabel=v["out_name"],
stdname=v.get("standard_name", ""),
long_name=v["long_name"],
units=v["units"],
modeling_realm=v["modeling_realm"],
frequency=freq,
mipTable=tbl,
cell_methods=v["cell_methods"],
cell_measures=v["cell_measures"],
positive=v["positive"],
Priority=float(v[mip_era.lower() +
"_priority"]),
label_without_psuffix=v["out_name"],
coordinates=v.get("dimensions", None))
dims = v["dimensions"].split(" ")
# get the index of time dimension to supress, if any
dr_dims = list()
all_dr_dims = list()
for (i, d) in enumerate(dims):
if "time" in d:
pass
elif d not in dr_single_levels:
dr_dims.append(d)
all_dr_dims.append(d)
else:
all_dr_dims.append(d)
drdims = "|".join(dr_dims)
if drdims in dims2shape:
extra_var.set_attributes(spatial_shp=dims2shape[drdims])
elif drdims.startswith("longitude|latitude|"):
# Allow the user to put any additional vertical dimension name
# which syntax fits further tests, such as P8MINE
edim = drdims.replace("longitude|latitude|", "", 1)
if any([
d.startswith("height") and d.endswith("m")
for d in edim.split("|")
]):
extra_var.set_attributes(spatial_shp="XY-HG")
else:
extra_var.set_attributes(spatial_shp='XY-' + edim)
if v["out_name"] not in dynamic_shapes[edim]:
dynamic_shapes[edim][v["out_name"]] = extra_var.spatial_shp
elif "spatial_shp" in v:
extra_var.set_attributes(spatial_shp=v["spatial_shp"])
else:
logger.warning(
"spatial shape corresponding to %s for variable %s in Table %s not found in DR."
% (drdims, v["out_name"], table))
dr_dimids = list()
for d in all_dr_dims:
if d in dim2dimid:
dr_dimids.append(dim2dimid[d])
extra_dim = get_simple_dim_from_dim_id(dim2dimid[d])
extra_var.update_attributes(
sdims={extra_dim.label: extra_dim})
else:
extra_dim_info = read_json_content(
json_coordinate)["axis_entry"]
if d in extra_dim_info:
extra_dim_info = extra_dim_info[d]
else:
raise KeyError(
"Could not find the dimension definition for %s in %s"
% (d, json_coordinate))
extra_dim = SimpleDim(
label=d,
axis=extra_dim_info["axis"],
stdname=extra_dim_info["standard_name"],
units=extra_dim_info["units"],
long_name=extra_dim_info["long_name"],
out_name=extra_dim_info["out_name"],
positive=extra_dim_info["positive"],
title=extra_dim_info.get("title",
extra_dim_info["long_name"]),
# values of multi vertical levels
requested=" ".join([
ilev for ilev in extra_dim_info["requested"]
]).rstrip(),
value=extra_dim_info[
"value"], # value of single vertical level
type=extra_dim_info["type"]) # axis type
extra_var.update_attributes(
sdims={extra_dim.label: extra_dim})
if v["out_name"] not in dim_from_extra:
dim_from_extra[d][v["out_name"]] = (
extra_dim.stdname, extra_dim.requested)
extra_var.set_attributes(label_without_psuffix=remove_p_suffix(
extra_var,
multi_plev_suffixes,
single_plev_suffixes,
realms=["atmos", "aerosol", "atmosChem"]))
extra_var.set_attributes(ref_var=extra_var.label_without_psuffix)
extravars.append(extra_var)
logger.info("For extra table %s (which has %d variables):" %
(table, len(extravars)))
logger.info(
"\tVariables which dim was found in extra coordinates table:\n%s" %
"\n".join([
"\t\t%20s: %s\n" % (d, " ".join(sorted(dim_from_extra[d])))
for d in sorted(list(dim_from_extra))
]))
logger.info(
"\tDynamical XY-xxx spatial shapes (shapes not found in DR):\n%s" %
"\n".join([
"\t\t%20s: %s\n" %
(("XY-" + d), " ".join(sorted(dynamic_shapes[d])))
for d in sorted(list(dynamic_shapes))
]))
return extravars
def read_home_vars_list(hmv_file, expid, mips, path_extra_tables=None):
"""
A function to get HOME variables that are not planned in the CMIP6 DataRequest but
the lab want to outpuut anyway
Args:
hmv_file (string) : a text file containing the list of home variables
expid (string) : if willing to filter on a given experiment
mips (string) : if willing to filter on given mips
path_extra_tables (string): path where to find extra Tables. Mandatory only if
there is'extra' lines in list of home variables.
Returns:
A list of 'simplified CMOR variables'
"""
logger = get_logger()
#
homevars_list = get_config_variable("homevars_list")
#
if hmv_file is None:
return list()
elif homevars_list is not None:
return homevars_list
else:
data = list()
file_list = [f for f in hmv_file.split(" ") if len(f) > 0]
for fil in file_list:
if not os.path.exists(fil):
raise VarsError(
"Abort: file for home variables does not exist: %s" % fil)
else:
# Read file
with open(fil, "r") as fp:
data.extend(fp.readlines())
data = [l.rstrip("\n\t ") for l in data if not l.startswith("#")]
data = [l for l in data if len(l) > 0]
# Build list of home variables
homevars = list()
extravars = list()
extra_vars_per_table = defaultdict(list)
for line in data:
line_split = line.split(";")
line_split = [
elt.strip(" ").lstrip("\n\t").strip(" ") for elt in line_split
]
line_split = [elt for elt in line_split if len(elt) > 0]
var_type = line_split[0]
if var_type in [
"cmor",
]:
home_var = read_home_var_cmor(line_split, mips, expid)
if home_var is not None:
homevars.append(home_var)
elif var_type in [
"perso",
]:
home_var = read_home_var_perso(line_split, mips, expid)
if home_var is not None:
homevars.append(home_var)
elif var_type in [
"extra",
]:
home_var = read_home_var_extra(
line_split,
expid,
mips,
path_extra_tables=path_extra_tables,
extra_vars_per_table=extra_vars_per_table)
extravars.extend(home_var)
elif var_type in [
"dev",
]:
home_var = read_home_var_dev(line_split, mips, expid)
if home_var is not None:
homevars.append(home_var)
else:
raise ValueError("Unknown type for var: %s (%s)" %
(var_type, line))
logger.info(
"Number of 'cmor', 'dev' and 'perso' among home variables: %d" %
len(homevars))
logger.info("Number of 'extra' among home variables: %d" %
len(extravars))
homevars.extend(extravars)
set_config_variable("homevars_list", homevars)
return homevars