def test_load_ovars():
ece2cmorlib.initialize()
try:
taskloader.load_targets({"Omon": ["tossq", "so", "thetao"], "Oday": ["sos"]})
eq_(len(ece2cmorlib.tasks), 4)
finally:
ece2cmorlib.finalize()
def test_load_expressions():
ece2cmorlib.initialize()
try:
taskloader.load_targets({"day": ["sfcWindmax"]})
eq_("var214=sqrt(sqr(var165)+sqr(var166))", getattr(ece2cmorlib.tasks[0].source, "expr"))
finally:
ece2cmorlib.finalize()
def test_load_avars():
ece2cmorlib.initialize()
try:
taskloader.load_targets({"3hr": ["clt", "uas", "vas"], "Amon": ["vas", "tas"]})
eq_(len(ece2cmorlib.tasks), 5)
eq_(2, len([t.source.get_grib_code().var_id for t in ece2cmorlib.tasks if t.target.variable == "vas"]))
finally:
ece2cmorlib.finalize()
def splitvars(varlist,ngroups,modlevs,tabid):
ece2cmorlib.initialize_without_cmor(tableprefix = tabid)
taskloader.load_targets(varlist)
fname = os.path.splitext(os.path.basename(varlist))[0]
taskgroups = balance_tasks(ece2cmorlib.tasks,ngroups,modlevs)
i = 0
for tasklist in taskgroups:
i += 1
write_varlist(tasklist,fname + "_" + str(i) + ".json")
def test_load_clt(self):
ece2cmorlib.initialize()
try:
taskloader.load_targets({"3hr": ["clt"]})
eq_(len(ece2cmorlib.tasks), 1)
src = ece2cmorlib.tasks[0].source
eq_(src.get_grib_code().var_id, 164)
finally:
ece2cmorlib.finalize()
def test_load_oavars():
ece2cmorlib.initialize()
try:
taskloader.load_targets({"3hr": ["clt", "uas"], "Amon": ["vas", "tas"], "Omon": ["tossq"]})
eq_(len(ece2cmorlib.tasks), 5)
eq_(4, len([t for t in ece2cmorlib.tasks if isinstance(t.source, cmor_source.ifs_source)]))
eq_(1, len([t for t in ece2cmorlib.tasks if isinstance(t.source, cmor_source.netcdf_source)]))
finally:
ece2cmorlib.finalize()
def test_load_unit_conv():
ece2cmorlib.initialize()
try:
taskloader.load_targets({"Amon": ["prc", "rsus", "zg"]})
eq_(len(ece2cmorlib.tasks), 3)
prctask = [t for t in ece2cmorlib.tasks if t.target.variable == "prc"][0]
rsustask = [t for t in ece2cmorlib.tasks if t.target.variable == "rsus"][0]
zgtask = [t for t in ece2cmorlib.tasks if t.target.variable == "zg"][0]
eq_("vol2flux", getattr(prctask, cmor_task.conversion_key))
eq_("cum2inst", getattr(rsustask, cmor_task.conversion_key))
eq_("pot2alt", getattr(zgtask, cmor_task.conversion_key))
finally:
ece2cmorlib.finalize()
def main():
parser = argparse.ArgumentParser(
description="Create IFS ppt files for given data request")
parser.add_argument(
"--vars",
metavar="FILE",
type=str,
required=True,
help=
"File (json|f90 namelist|xlsx) containing cmor variables (Required)")
parser.add_argument("--tabdir",
metavar="DIR",
type=str,
default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tabid",
metavar="PREFIX",
type=str,
default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
args = parser.parse_args()
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default,
mode=ece2cmorlib.PRESERVE,
tabledir=args.tabdir,
tableprefix=args.tabid)
# Load the variables as task targets:
taskloader.load_targets(args.vars,
active_components={
"ifs": True,
"nemo": False
})
# Write the IFS input files
write_ppt_files(ece2cmorlib.tasks)
# Finishing up
ece2cmorlib.finalize_without_cmor()
def main():
parser = argparse.ArgumentParser(description = "Validate input variable list against CMIP tables")
parser.add_argument("--vars", metavar = "FILE", type = str, required = True, help = "File (json|f90 namelist|xlsx) containing cmor variables (Required)")
parser.add_argument("--tabdir", metavar = "DIR", type = str, default = ece2cmorlib.table_dir_default, help = "Cmorization table directory")
parser.add_argument("--tabid", metavar = "PREFIX", type = str, default = ece2cmorlib.prefix_default, help = "Cmorization table prefix string")
parser.add_argument("--output", metavar = "FILE", type = str, default = None, help = "Output path to write variables to")
parser.add_argument("--withouttablescheck", action = "store_true", default = False, help = "Ignore variable tables when performing var checking")
parser.add_argument("-v", "--verbose", action = "store_true", default = False, help = "Write xlsx and ASCII files with verbose output (suppress the related terminal messages as the content of these files contain this information)")
parser.add_argument("-a", "--atm", action = "store_true", default = False, help = "Run exclusively for atmosphere variables")
parser.add_argument("-o", "--oce", action = "store_true", default = False, help = "Run exclusively for ocean variables")
args = parser.parse_args()
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default,mode = ece2cmorlib.PRESERVE,tabledir = args.tabdir,tableprefix = args.tabid)
# Fix conflicting flags
procatmos,prococean = not args.oce,not args.atm
if(not procatmos and not prococean):
procatmos,prococean = True,True
# Configure task loader:
taskloader.skip_tables = args.withouttablescheck
# Load the variables as task targets:
loadedtargets,ignoredtargets,identifiedmissingtargets,missingtargets = taskloader.load_targets(args.vars,load_atm_tasks = procatmos,load_oce_tasks = prococean, silent = args.verbose)
if(args.output):
output_dir = os.path.dirname(args.output)
if(not os.path.isdir(output_dir)):
if(output_dir != ''): os.makedirs(output_dir)
write_varlist(loadedtargets,args.output + ".available.json")
if(args.verbose):
write_varlist_ascii(loadedtargets ,args.output + ".available.txt")
write_varlist_ascii(ignoredtargets ,args.output + ".ignored.txt")
write_varlist_ascii(identifiedmissingtargets,args.output + ".identifiedmissing.txt")
write_varlist_ascii(missingtargets ,args.output + ".missing.txt")
write_varlist_excel(loadedtargets ,args.output + ".available.xlsx")
write_varlist_excel(ignoredtargets ,args.output + ".ignored.xlsx")
write_varlist_excel(identifiedmissingtargets,args.output + ".identifiedmissing.xlsx")
write_varlist_excel(missingtargets ,args.output + ".missing.xlsx")
# Finishing up
ece2cmorlib.finalize_without_cmor()
def main(args = None):
if args is None:
args = sys.argv[1:]
varlist_path_default = os.path.join(os.path.dirname(__file__),"resources","varlist.json")
parser = argparse.ArgumentParser(description = "Post-processing and cmorization of EC-Earth output",
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("datadir", metavar = "DIR", type = str)
parser.add_argument("date", metavar = "YYYY-mm-dd", type = str)
parser.add_argument("--vars", metavar = "FILE" , type = str, default = varlist_path_default,help = "File (json|f90 namelist|xlsx) containing cmor variables")
parser.add_argument("--conf", metavar = "FILE.json", type = str, default = ece2cmorlib.conf_path_default,help = "Input metadata file")
parser.add_argument("--exp", metavar = "EXPID", type = str, default = "ECE3",help = "Experiment prefix")
parser.add_argument("--refd", metavar = "YYYY-mm-dd", type = str, default = None,help = "Reference date (for atmosphere data), by default the start date")
parser.add_argument("--mode", metavar = "MODE", type = str, default = "preserve",help = "CMOR netcdf mode",choices = ["preserve","replace","append"])
parser.add_argument("--freq", metavar = "N", type = int, default = 3,help = "IFS output frequency, in hours (not required if autofilter is used)")
parser.add_argument("--tabdir", metavar = "DIR", type = str, default = ece2cmorlib.table_dir_default,help = "Cmorization table directory")
parser.add_argument("--tabid", metavar = "PREFIX", type = str, default = ece2cmorlib.prefix_default,help = "Cmorization table prefix string")
parser.add_argument("--tmpdir", metavar = "DIR", type = str, default = "/tmp/ece2cmor3", help = "Temporary working directory")
parser.add_argument("--npp", metavar = "N", type = int, default = 8, help = "Number of post-processing threads")
parser.add_argument("--tmpsize",metavar = "X", type = float, default = float("inf"),help = "Size of tempdir (in GB) that triggers flushing")
parser.add_argument("--ncdo", metavar = "N", type = int, default = 4,help = "Number of available threads per CDO postprocessing task")
parser.add_argument("-a", "--atm", action = "store_true", default = False, help = "Run ece2cmor3 exclusively for atmosphere data")
parser.add_argument("-o", "--oce", action = "store_true", default = False, help = "Run ece2cmor3 exclusively for ocean data")
parser.add_argument("--nomask" , action = "store_true", default = False, help = "Disable masking of fields")
parser.add_argument("--filter" , action = "store_true", default = False, help = "Automatic filtering of grib files")
args = parser.parse_args()
modedict = {"preserve":ece2cmorlib.PRESERVE,"append":ece2cmorlib.APPEND,"replace":ece2cmorlib.REPLACE}
# Initialize ece2cmor:
ece2cmorlib.initialize(args.conf,mode = modedict[args.mode],tabledir = args.tabdir,tableprefix = args.tabid)
ece2cmorlib.enable_masks = not args.nomask
ece2cmorlib.auto_filter = args.filter
# Fix conflicting flags
procatmos,prococean = not args.oce,not args.atm
if(not procatmos and not prococean):
procatmos,prococean = True,True
# Load the variables as task targets:
taskloader.load_targets(args.vars,load_atm_tasks = procatmos,load_oce_tasks = prococean)
startdate = dateutil.parser.parse(args.date)
length = dateutil.relativedelta.relativedelta(months = 1)
if(procatmos):
refdate = dateutil.parser.parse(args.refd) if args.refd else None
# Execute the atmosphere cmorization:
ece2cmorlib.perform_ifs_tasks(args.datadir,args.exp,startdate,length,refdate = refdate,
outputfreq = args.freq,
tempdir = args.tmpdir,
taskthreads = args.npp,
cdothreads = args.ncdo,
maxsizegb = args.tmpsize)
if(prococean):
ece2cmorlib.perform_nemo_tasks(args.datadir,args.exp,startdate,length)
ece2cmorlib.finalize()
def main():
parser = argparse.ArgumentParser(description="Create NEMO XIOS file_def input files for a given CMIP6 data request")
parser.add_argument("--vars", metavar="FILE", type=str, required=True,
help="File (json|f90 namelist|xlsx) containing cmor variables (Required)")
parser.add_argument("--tabdir", metavar="DIR", type=str, default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tabid", metavar="PREFIX", type=str, default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
args = parser.parse_args()
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default, mode=ece2cmorlib.PRESERVE, tabledir=args.tabdir,
tableprefix=args.tabid)
# Load the variables as task targets:
taskloader.load_targets(args.vars, active_components={"ifs": False, "nemo": True})
for task in ece2cmorlib.tasks:
print ' {:15} {:8} {:15} {}'.format(task.target.variable, task.target.table, task.target.units, task.target.frequency)
#print task.target.__dict__
print ' Number of activated data request tasks is', len(ece2cmorlib.tasks)
# READING THE BASIC FILE_DEF FILE:
if os.path.isfile(basic_file_def_file_name) == False: print ' The file ', basic_file_def_file_name, ' does not exist.'; sys.exit(' stop')
tree_basic_file_def = xmltree.parse(basic_file_def_file_name)
root_basic_file_def = tree_basic_file_def.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
#field_elements_basic_file_def = root_basic_file_def[0][:]
count = 0
for field in root_basic_file_def.findall('.//field[@id]'):
for task in ece2cmorlib.tasks:
if field.attrib["name"] == task.target.variable and field.attrib["table"] == task.target.table:
field.attrib["enabled"] = "True"
count = count + 1
#print field.attrib["name"], field.attrib["table"]
# After the table attribute has been used to match with the data request, the table attribute is removed here because it is not a valid XIOS attribute:
field.attrib.pop('table', None)
# Write the NEMO XIOS file_def input files:
tree_basic_file_def.write(file_def_file_name)
print ' The number of variables which is enabled in', file_def_file_name, ' is', count
# SPLIT THE FILE_DEF FILE IN THREE FILE_DEF FILES FOR OPA, LIM AND PISCES:
# FILE_DEF FILE FOR OPA:
tree_opa = xmltree.parse(file_def_file_name)
root_opa = tree_opa.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
for file_element in root_opa.findall('./file_group/file'):
# Get the model component info from this attribute by using a regular expression:
model_component = re.search('_(.+?)_', file_element.attrib["name_suffix"]).group(1)
if model_component == 'lim' or model_component == 'pisces':
#print ' Remove file for opa file_def:', file_element.attrib["id"], model_component
# Remove this file element from its parent element the file_group element:
root_opa[0].remove(file_element)
root_opa[0].attrib["id"] = "id_file_group_opa"
tree_opa.write(file_def_opa_file_name)
# FILE_DEF FILE FOR LIM:
tree_lim = xmltree.parse(file_def_file_name)
root_lim = tree_lim.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
for file_element in root_lim.findall('./file_group/file'):
# Get the model component info from this attribute by using a regular expression:
model_component = re.search('_(.+?)_', file_element.attrib["name_suffix"]).group(1)
if model_component == 'opa' or model_component == 'pisces':
#print ' Remove file for lim file_def:', file_element.attrib["id"], model_component
# Remove this file element from its parent element the file_group element:
root_lim[0].remove(file_element)
root_lim[0].attrib["id"] = "id_file_group_lim"
tree_lim.write(file_def_lim_file_name)
# FILE_DEF FILE FOR PISCES:
tree_pisces = xmltree.parse(file_def_file_name)
root_pisces = tree_pisces.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
for file_element in root_pisces.findall('./file_group/file'):
# Get the model component info from this attribute by using a regular expression:
model_component = re.search('_(.+?)_', file_element.attrib["name_suffix"]).group(1)
if model_component == 'opa' or model_component == 'lim':
#print ' Remove file for pisces file_def:', file_element.attrib["id"], model_component
# Remove this file element from its parent element the file_group element:
root_pisces[0].remove(file_element)
root_pisces[0].attrib["id"] = "id_file_group_pisces"
tree_pisces.write(file_def_pisces_file_name)
# Finishing up
ece2cmorlib.finalize_without_cmor()
# PRODUCE FILE_DEF FILES FOR OPA, LIM AND PISCES WITH ONLY ENABLED VARIABLES:
# FILE_DEF FILE FOR OPA WITH ONLY ENABLED VARIABLES:
tree_opa_enabled_only = xmltree.parse(file_def_opa_file_name)
root_opa_enabled_only = tree_opa_enabled_only.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
for file_element in root_opa_enabled_only.findall('./file_group/file'):
for field_element in file_element.findall('field[@enabled="False"]'): file_element.remove(field_element)
tree_opa_enabled_only.write(file_def_opa_file_name_compact)
# FILE_DEF FILE FOR LIM WITH ONLY ENABLED VARIABLES:
tree_lim_enabled_only = xmltree.parse(file_def_lim_file_name)
root_lim_enabled_only = tree_lim_enabled_only.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
for file_element in root_lim_enabled_only.findall('./file_group/file'):
for field_element in file_element.findall('field[@enabled="False"]'): file_element.remove(field_element)
tree_lim_enabled_only.write(file_def_lim_file_name_compact)
# FILE_DEF FILE FOR PISCES WITH ONLY ENABLED VARIABLES:
tree_pisces_enabled_only = xmltree.parse(file_def_pisces_file_name)
root_pisces_enabled_only = tree_pisces_enabled_only.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
for file_element in root_pisces_enabled_only.findall('./file_group/file'):
for field_element in file_element.findall('field[@enabled="False"]'): file_element.remove(field_element)
tree_pisces_enabled_only.write(file_def_pisces_file_name_compact)
def main(args=None):
if args is None:
pass
varlist_path_default = os.path.join(os.path.dirname(__file__), "resources",
"varlist.json")
parser = argparse.ArgumentParser(
description="Post-processing and cmorization of EC-Earth output",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("datadir", metavar="DIR", type=str)
parser.add_argument("date", metavar="YYYY-mm-dd", type=str)
parser.add_argument(
"--vars",
metavar="FILE",
type=str,
default=varlist_path_default,
help="File (json|f90 namelist|xlsx) containing cmor variables")
parser.add_argument("--conf",
metavar="FILE.json",
type=str,
default=ece2cmorlib.conf_path_default,
help="Input metadata file")
parser.add_argument("--exp",
metavar="EXPID",
type=str,
default="ECE3",
help="Experiment prefix")
parser.add_argument(
"--refd",
metavar="YYYY-mm-dd",
type=str,
default=None,
help="Reference date (for atmosphere data), by default the start date")
parser.add_argument("--mode",
metavar="MODE",
type=str,
default="preserve",
help="CMOR netcdf mode",
choices=["preserve", "replace", "append"])
parser.add_argument(
"--freq",
metavar="N",
type=int,
default=3,
help=
"IFS output frequency, in hours (not required if autofilter is used)")
parser.add_argument("--tabdir",
metavar="DIR",
type=str,
default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tabid",
metavar="PREFIX",
type=str,
default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
parser.add_argument("--tmpdir",
metavar="DIR",
type=str,
default="/tmp/ece2cmor",
help="Temporary working directory")
parser.add_argument("--npp",
metavar="N",
type=int,
default=8,
help="Number of post-processing threads")
parser.add_argument("--tmpsize",
metavar="X",
type=float,
default=float("inf"),
help="Size of tempdir (in GB) that triggers flushing")
parser.add_argument(
"--ncdo",
metavar="N",
type=int,
default=4,
help="Number of available threads per CDO postprocessing task")
parser.add_argument("--nomask",
action="store_true",
default=False,
help="Disable masking of fields")
parser.add_argument("--filter",
action="store_true",
default=False,
help="Automatic filtering of grib files")
model_attributes, model_tabfile_attributes = {}, {}
for c in components.models:
flag1, flag2 = components.get_script_options(c)
if flag2 is None:
parser.add_argument("--" + flag1,
action="store_true",
default=False,
help="Run ece2cmor3 exclusively for %s data" %
c)
else:
parser.add_argument("--" + flag1,
'-' + flag2,
action="store_true",
default=False,
help="Run ece2cmor3 exclusively for %s data" %
c)
model_attributes[c] = flag1
tabfile = components.models[c].get(components.table_file, "")
if tabfile:
option = os.path.basename(tabfile)
model_tabfile_attributes[c] = option
parser.add_argument("--" + option,
metavar="FILE.json",
type=str,
default=tabfile,
help="%s variable table (optional)" % c)
args = parser.parse_args()
modedict = {
"preserve": ece2cmorlib.PRESERVE,
"append": ece2cmorlib.APPEND,
"replace": ece2cmorlib.REPLACE
}
# Initialize ece2cmor:
ece2cmorlib.initialize(args.conf,
mode=modedict[args.mode],
tabledir=args.tabdir,
tableprefix=args.tabid)
ece2cmorlib.enable_masks = not args.nomask
ece2cmorlib.auto_filter = args.filter
# Fix exclusive run flags: if none are used, we cmorize for all components
model_active_flags = dict.fromkeys(components.models, False)
for model in model_attributes:
model_active_flags[model] = getattr(args, model_attributes[model],
False)
if not any(model_active_flags.values()):
model_active_flags = dict.fromkeys(model_active_flags, True)
# Load the variables as task targets:
for model in model_tabfile_attributes:
tabfile_attribute = model_tabfile_attributes[model]
attribute_value = getattr(args, tabfile_attribute, None)
if attribute_value is not None:
components.models[model][components.table_file] = attribute_value
taskloader.load_targets(args.vars, model_active_flags)
startdate = dateutil.parser.parse(args.date)
length = dateutil.relativedelta.relativedelta(months=1)
if model_active_flags["ifs"]:
refdate = dateutil.parser.parse(args.refd) if args.refd else None
# Execute the atmosphere cmorization:
ece2cmorlib.perform_ifs_tasks(args.datadir,
args.exp,
startdate,
length,
refdate=refdate,
outputfreq=args.freq,
tempdir=args.tmpdir,
taskthreads=args.npp,
cdothreads=args.ncdo,
maxsizegb=args.tmpsize)
if model_active_flags["nemo"]:
ece2cmorlib.perform_nemo_tasks(args.datadir, args.exp, startdate,
length)
# if procNEWCOMPONENT:
# ece2cmorlib.perform_NEWCOMPONENT_tasks(args.datadir, args.exp, startdate, length)
ece2cmorlib.finalize()
