def main():
parser = argparse.ArgumentParser(description = "Check for obsolete cmor variables in json file")
parser.add_argument("--tabdir", metavar = "DIR", type = str, default = ece2cmorlib.table_dir_default, help = "Cmorization table directory")
parser.add_argument("--tabid", metavar = "PREFIX", type = str, default = ece2cmorlib.prefix_default, help = "Cmorization table prefix string")
parser.add_argument("--output", metavar = "FILE", type = str, default = None, help = "Output path to write variables to")
parser.add_argument("-a", "--atm", action = "store_true", default = False, help = "Run exclusively for atmosphere variables")
parser.add_argument("-o", "--oce", action = "store_true", default = False, help = "Run exclusively for ocean variables")
args = parser.parse_args()
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default,mode = ece2cmorlib.PRESERVE,tabledir = args.tabdir,tableprefix = args.tabid)
# Fix conflicting flags
procatmos,prococean = not args.oce,not args.atm
if(not procatmos and not prococean):
procatmos,prococean = True,True
print
if(prococean):
fname = "../resources/nemopar.json"
check_obsolete(fname)
print
if(procatmos):
fname = "../resources/ifspar.json"
check_obsolete(fname)
# Finishing up
ece2cmorlib.finalize_without_cmor()
def test_load_expressions():
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq({"day": ["sfcWindmax"]})
assert getattr(ece2cmorlib.tasks[0].source,
"expr") == "var214=sqrt(sqr(var165)+sqr(var166))"
finally:
ece2cmorlib.finalize_without_cmor()
def splitvars(varlist,ngroups,modlevs,tabid):
ece2cmorlib.initialize_without_cmor(tableprefix = tabid)
taskloader.load_targets(varlist)
fname = os.path.splitext(os.path.basename(varlist))[0]
taskgroups = balance_tasks(ece2cmorlib.tasks,ngroups,modlevs)
i = 0
for tasklist in taskgroups:
i += 1
write_varlist(tasklist,fname + "_" + str(i) + ".json")
def test_load_oclim_variable():
ece2cmorlib.initialize_without_cmor()
try:
matches, omitted = taskloader.load_drq({"Oclim": ["difvho"]},
check_prefs=False)
assert len(matches["nemo"]) == 1
assert not any(omitted)
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_ovars():
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq({
"Omon": ["tossq", "so", "thetao"],
"Oday": ["sos"]
})
assert len(ece2cmorlib.tasks) == 4
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_clt():
ece2cmorlib.initialize_without_cmor()
try:
clt3hr = {"3hr": ["clt"]}
taskloader.load_tasks_from_drq(clt3hr)
assert len(ece2cmorlib.tasks) == 1
src = ece2cmorlib.tasks[0].source
assert src.get_grib_code().var_id == 164
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_cdnc_variable():
ece2cmorlib.initialize_without_cmor()
try:
tasks = taskloader.load_tasks({"ifs": {"AERmon": ["cdnc"]}})
eq_(len(tasks), 1)
src = tasks[0].source
ok_(isinstance(src, cmor_source.ifs_source))
eq_(getattr(src, "expr_order", 0), 1)
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_clt():
ece2cmorlib.initialize_without_cmor()
try:
clt3hr = {"3hr": ["clt"]}
taskloader.load_tasks_from_drq(clt3hr)
eq_(len(ece2cmorlib.tasks), 1)
src = ece2cmorlib.tasks[0].source
eq_(src.get_grib_code().var_id, 164)
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_cdnc_variable():
ece2cmorlib.initialize_without_cmor()
try:
tasks = taskloader.load_tasks({"ifs": {"AERmon": ["cdnc"]}})
assert len(tasks) == 1
src = tasks[0].source
assert isinstance(src, cmor_source.ifs_source)
assert getattr(src, "expr_order", 0) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_cfc12_Omon_prefs():
ece2cmorlib.initialize_without_cmor()
try:
matches, omitted = taskloader.load_drq({"Omon": ["cfc12"]},
config="EC-EARTH-AOGCM")
assert not any(matches["nemo"])
matches, omitted = taskloader.load_drq({"Omon": ["cfc12"]},
config="EC-EARTH-CC")
assert len(matches["nemo"]) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_use_zg_preferences():
ece2cmorlib.initialize_without_cmor()
try:
matches, omitted = taskloader.load_drq(
{"6hrPlevPt": ["zg7h", "zg27"]}, check_prefs=False)
assert len(matches["ifs"]) == 2
matches, omitted = taskloader.load_drq(
{"6hrPlevPt": ["zg7h", "zg27"]}, check_prefs=True)
assert len(matches["ifs"]) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_use_level_preferences():
ece2cmorlib.initialize_without_cmor()
try:
matches, omitted = taskloader.load_drq(
{"6hrPlevPt": ["ua", "ua7h"]}, check_prefs=False)
eq_(len(matches["ifs"]), 2)
matches, omitted = taskloader.load_drq(
{"6hrPlevPt": ["ua", "ua7h"]}, check_prefs=True)
eq_(len(matches["ifs"]), 1)
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_script_variable():
ece2cmorlib.initialize_without_cmor()
try:
tasks = taskloader.load_tasks({"ifs": {"EmonZ": ["epfy"]}})
assert len(tasks) == 1
src = tasks[0].source
assert isinstance(src, cmor_source.ifs_source)
script = getattr(tasks[0], "post-proc", None)
assert script in ece2cmorlib.scripts.keys(
) and ece2cmorlib.scripts[script]["component"] == "ifs"
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_clt_json():
ece2cmorlib.initialize_without_cmor()
try:
clt3hr = {"3hr": ["clt"]}
taskloader_test.setup_drq(clt3hr)
taskloader.load_tasks_from_drq(drqpath)
assert len(ece2cmorlib.tasks) == 1
src = ece2cmorlib.tasks[0].source
assert src.get_grib_code().var_id == 164
finally:
taskloader_test.cleanup_drq()
ece2cmorlib.finalize_without_cmor()
def main():
parser = argparse.ArgumentParser(description="Create IFS ppt files for given data request")
varsarg = parser.add_mutually_exclusive_group(required=True)
varsarg.add_argument("--vars", metavar="FILE", type=str,
help="File (json) containing cmor variables per EC-Earth component")
varsarg.add_argument("--drq", metavar="FILE", type=str,
help="File (json|f90 namelist|xlsx) containing cmor variables")
varsarg.add_argument("--allvars", action="store_true", default=False,
help="Read all possible variables from CMOR tables")
parser.add_argument("--tabdir", metavar="DIR", type=str, default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tabid", metavar="PREFIX", type=str, default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
args = parser.parse_args()
print()
print('Running drq2ppt with:')
print(' drq2ppt ' + cmor_utils.ScriptUtils.get_drq_vars_options(args))
print()
if args.vars is not None and not os.path.isfile(args.vars):
log.fatal("Error: Your variable list json file %s cannot be found." % args.vars)
sys.exit(' Exiting drq2ppt.')
if args.drq is not None and not os.path.isfile(args.drq):
log.fatal("Error: Your data request file %s cannot be found." % args.drq)
sys.exit(' Exiting drq2ppt.')
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default, mode=ece2cmorlib.PRESERVE, tabledir=args.tabdir,
tableprefix=args.tabid)
# Load only atmosphere variables as task targets:
try:
if getattr(args, "vars", None) is not None:
taskloader.load_tasks(args.vars, active_components=["ifs"])
elif getattr(args, "allvars", False):
taskloader.load_tasks_from_drq("allvars", active_components=["ifs"], check_prefs=False)
else:
taskloader.load_tasks_from_drq(args.drq, active_components=["ifs"], check_prefs=False)
except taskloader.SwapDrqAndVarListException as e:
log.error(e.message)
opt1, opt2 = "vars" if e.reverse else "drq", "drq" if e.reverse else "vars"
log.error("It seems you are using the --%s option where you should use the --%s option for this file"
% (opt1, opt2))
sys.exit(' Exiting drq2ppt.')
# Write the IFS input files
write_ppt_files(ece2cmorlib.tasks)
# Finishing up
ece2cmorlib.finalize_without_cmor()
def test_load_ps_AERmon_prefs():
ece2cmorlib.initialize_without_cmor()
try:
matches, omitted = taskloader.load_drq({"AERmon": ["ps"]},
check_prefs=False)
assert len(matches["ifs"]) == 1
assert len(matches["tm5"]) == 1
matches, omitted = taskloader.load_drq({"AERmon": ["ps"]},
check_prefs=True)
assert len(matches["ifs"]) == 0
assert len(matches["tm5"]) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_ignored_variables():
ece2cmorlib.initialize_without_cmor()
try:
matches, omissions = taskloader.load_drq({
"Emon": ["rsdsdiff"],
"Amon": ["clivi"]
})
ignored, identified_missing, missing, dismissed = taskloader.split_targets(
omissions)
assert len(matches["nemo"]) == 0
assert len(matches["ifs"]) == 1
assert len(ignored) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_avars_drq():
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq({
"3hr": ["clt", "uas", "vas"],
"Amon": ["vas", "tas"]
})
assert len(ece2cmorlib.tasks) == 5
assert len([
t.source.get_grib_code().var_id for t in ece2cmorlib.tasks
if t.target.variable == "vas"
]) == 2
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_avars_drq_json():
ece2cmorlib.initialize_without_cmor()
try:
avars = {"3hr": ["clt", "uas", "vas"], "Amon": ["vas", "tas"]}
taskloader_test.setup_drq(avars)
taskloader.load_tasks_from_drq(drqpath)
eq_(len(ece2cmorlib.tasks), 5)
eq_(
2,
len([
t.source.get_grib_code().var_id for t in ece2cmorlib.tasks
if t.target.variable == "vas"
]))
finally:
taskloader_test.cleanup_drq()
ece2cmorlib.finalize_without_cmor()
def test_load_tsl_table_override():
ece2cmorlib.initialize_without_cmor()
try:
tasks = taskloader.load_tasks_from_drq({
"6hrPlevPt": ["tsl"],
"Lmon": ["tsl"]
})
eq_(len(tasks), 2)
for t in tasks:
if t.target.table == "6hrPlevPt":
ok_(not hasattr(t.source, "expr"))
ok_(t.source.get_grib_code() == cmor_source.grib_code(139))
else:
ok_("merge" in getattr(t.source, "expr"))
finally:
ece2cmorlib.finalize_without_cmor()
def main():
parser = argparse.ArgumentParser(description = "Validate input variable list against CMIP tables")
parser.add_argument("--vars", metavar = "FILE", type = str, required = True, help = "File (json|f90 namelist|xlsx) containing cmor variables (Required)")
parser.add_argument("--tabdir", metavar = "DIR", type = str, default = ece2cmorlib.table_dir_default, help = "Cmorization table directory")
parser.add_argument("--tabid", metavar = "PREFIX", type = str, default = ece2cmorlib.prefix_default, help = "Cmorization table prefix string")
parser.add_argument("--output", metavar = "FILE", type = str, default = None, help = "Output path to write variables to")
parser.add_argument("--withouttablescheck", action = "store_true", default = False, help = "Ignore variable tables when performing var checking")
parser.add_argument("-v", "--verbose", action = "store_true", default = False, help = "Write xlsx and ASCII files with verbose output (suppress the related terminal messages as the content of these files contain this information)")
parser.add_argument("-a", "--atm", action = "store_true", default = False, help = "Run exclusively for atmosphere variables")
parser.add_argument("-o", "--oce", action = "store_true", default = False, help = "Run exclusively for ocean variables")
args = parser.parse_args()
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default,mode = ece2cmorlib.PRESERVE,tabledir = args.tabdir,tableprefix = args.tabid)
# Fix conflicting flags
procatmos,prococean = not args.oce,not args.atm
if(not procatmos and not prococean):
procatmos,prococean = True,True
# Configure task loader:
taskloader.skip_tables = args.withouttablescheck
# Load the variables as task targets:
loadedtargets,ignoredtargets,identifiedmissingtargets,missingtargets = taskloader.load_targets(args.vars,load_atm_tasks = procatmos,load_oce_tasks = prococean, silent = args.verbose)
if(args.output):
output_dir = os.path.dirname(args.output)
if(not os.path.isdir(output_dir)):
if(output_dir != ''): os.makedirs(output_dir)
write_varlist(loadedtargets,args.output + ".available.json")
if(args.verbose):
write_varlist_ascii(loadedtargets ,args.output + ".available.txt")
write_varlist_ascii(ignoredtargets ,args.output + ".ignored.txt")
write_varlist_ascii(identifiedmissingtargets,args.output + ".identifiedmissing.txt")
write_varlist_ascii(missingtargets ,args.output + ".missing.txt")
write_varlist_excel(loadedtargets ,args.output + ".available.xlsx")
write_varlist_excel(ignoredtargets ,args.output + ".ignored.xlsx")
write_varlist_excel(identifiedmissingtargets,args.output + ".identifiedmissing.xlsx")
write_varlist_excel(missingtargets ,args.output + ".missing.xlsx")
# Finishing up
ece2cmorlib.finalize_without_cmor()
def test_load_oavars_ocean():
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq(
{
"3hr": ["clt", "uas"],
"Amon": ["vas", "tas"],
"Omon": ["tossq"]
},
active_components=["nemo"])
assert len(ece2cmorlib.tasks) == 1
assert len([
t for t in ece2cmorlib.tasks
if isinstance(t.source, cmor_source.netcdf_source)
]) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_avars():
ece2cmorlib.initialize_without_cmor()
try:
avars = {
"ifs": {
"3hr": ["clt", "uas", "vas"],
"Amon": ["vas", "tas"]
}
}
taskloader.load_tasks(avars)
eq_(len(ece2cmorlib.tasks), 5)
eq_(
2,
len([
t.source.get_grib_code().var_id for t in ece2cmorlib.tasks
if t.target.variable == "vas"
]))
finally:
ece2cmorlib.finalize_without_cmor()
def test_dismiss_duplicates():
ece2cmorlib.initialize_without_cmor()
try:
matches, omitted = taskloader.load_drq({"AERmon": ["ps"]},
check_prefs=False)
assert len(matches["ifs"]) == 1
assert len(matches["tm5"]) == 1
tasks = taskloader.load_tasks(matches,
active_components=["ifs"],
target_filters=None,
check_duplicates=True)
assert not any(tasks)
tasks = taskloader.load_tasks(matches,
active_components=["ifs"],
target_filters=None,
check_duplicates=False)
assert len(tasks) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_skip_alevel():
def ifs_model_level_variable(target):
zaxis, levs = cmor_target.get_z_axis(target)
return zaxis not in ["alevel", "alevhalf"]
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq(
{
"3hr": ["clt", "uas", "vas"],
"CFmon": ["clwc", "hur", "ps"]
},
target_filters={"model level": ifs_model_level_variable})
assert len(ece2cmorlib.tasks) == 4
assert len([
t for t in ece2cmorlib.tasks if t.target.table == "CFmon"
]) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_avars_json():
ece2cmorlib.initialize_without_cmor()
try:
avars = {
"ifs": {
"3hr": ["clt", "uas", "vas"],
"Amon": ["vas", "tas"]
}
}
taskloader_test.setup_drq(avars)
taskloader.load_tasks(drqpath)
assert len(ece2cmorlib.tasks) == 5
assert len([
t.source.get_grib_code().var_id for t in ece2cmorlib.tasks
if t.target.variable == "vas"
]) == 2
finally:
taskloader_test.cleanup_drq()
ece2cmorlib.finalize_without_cmor()
def test_load_oavars():
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq({
"3hr": ["clt", "uas"],
"Amon": ["vas", "tas"],
"Omon": ["tossq"]
})
assert len(ece2cmorlib.tasks) == 5
assert len([
t for t in ece2cmorlib.tasks
if isinstance(t.source, cmor_source.ifs_source)
]) == 4
assert len([
t for t in ece2cmorlib.tasks
if isinstance(t.source, cmor_source.netcdf_source)
]) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_unit_conv():
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq({"Amon": ["prc", "rsus", "zg"]})
assert len(ece2cmorlib.tasks) == 3
prctask = [
t for t in ece2cmorlib.tasks if t.target.variable == "prc"
][0]
rsustask = [
t for t in ece2cmorlib.tasks if t.target.variable == "rsus"
][0]
zgtask = [
t for t in ece2cmorlib.tasks if t.target.variable == "zg"
][0]
assert getattr(prctask, cmor_task.conversion_key) == "vol2flux"
assert getattr(rsustask, cmor_task.conversion_key) == "cum2inst"
assert getattr(zgtask, cmor_task.conversion_key) == "pot2alt"
finally:
ece2cmorlib.finalize_without_cmor()
def main():
parser = argparse.ArgumentParser(
description="Create IFS ppt files for given data request")
parser.add_argument(
"--vars",
metavar="FILE",
type=str,
required=True,
help=
"File (json|f90 namelist|xlsx) containing cmor variables (Required)")
parser.add_argument("--tabdir",
metavar="DIR",
type=str,
default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tabid",
metavar="PREFIX",
type=str,
default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
args = parser.parse_args()
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default,
mode=ece2cmorlib.PRESERVE,
tabledir=args.tabdir,
tableprefix=args.tabid)
# Load the variables as task targets:
taskloader.load_targets(args.vars,
active_components={
"ifs": True,
"nemo": False
})
# Write the IFS input files
write_ppt_files(ece2cmorlib.tasks)
# Finishing up
ece2cmorlib.finalize_without_cmor()
def main():
parser = argparse.ArgumentParser(description="Check for obsolete cmor variables in json file")
parser.add_argument("--tabdir", metavar="DIR", type=str, default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tabid", metavar="PREFIX", type=str, default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
parser.add_argument("--output", metavar="FILE", type=str, default=None, help="Output path to write variables to")
cmor_utils.ScriptUtils.add_model_exclusive_options(parser, "checkvars")
args = parser.parse_args()
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default, mode=ece2cmorlib.PRESERVE, tabledir=args.tabdir,
tableprefix=args.tabid)
active_components = cmor_utils.ScriptUtils.get_active_components(args)
for model in components.models.keys():
if model in active_components:
check_obsolete(components.models[model][components.table_file])
# Finishing up
ece2cmorlib.finalize_without_cmor()