def test_load_expressions():
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq({"day": ["sfcWindmax"]})
assert getattr(ece2cmorlib.tasks[0].source,
"expr") == "var214=sqrt(sqr(var165)+sqr(var166))"
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_clt():
ece2cmorlib.initialize_without_cmor()
try:
clt3hr = {"3hr": ["clt"]}
taskloader.load_tasks_from_drq(clt3hr)
eq_(len(ece2cmorlib.tasks), 1)
src = ece2cmorlib.tasks[0].source
eq_(src.get_grib_code().var_id, 164)
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_ovars():
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq({
"Omon": ["tossq", "so", "thetao"],
"Oday": ["sos"]
})
assert len(ece2cmorlib.tasks) == 4
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_clt():
ece2cmorlib.initialize_without_cmor()
try:
clt3hr = {"3hr": ["clt"]}
taskloader.load_tasks_from_drq(clt3hr)
assert len(ece2cmorlib.tasks) == 1
src = ece2cmorlib.tasks[0].source
assert src.get_grib_code().var_id == 164
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_tos_3hr():
ece2cmorlib.initialize()
try:
taskloader.load_tasks_from_drq({"3hr": ["tos"]},
active_components=["ifs"])
assert not any(ece2cmorlib.tasks)
taskloader.load_tasks_from_drq({"3hr": ["tos"]},
active_components=["nemo"])
assert len(ece2cmorlib.tasks) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_clt_json():
ece2cmorlib.initialize_without_cmor()
try:
clt3hr = {"3hr": ["clt"]}
taskloader_test.setup_drq(clt3hr)
taskloader.load_tasks_from_drq(drqpath)
assert len(ece2cmorlib.tasks) == 1
src = ece2cmorlib.tasks[0].source
assert src.get_grib_code().var_id == 164
finally:
taskloader_test.cleanup_drq()
ece2cmorlib.finalize_without_cmor()
def main():
parser = argparse.ArgumentParser(description="Create IFS ppt files for given data request")
varsarg = parser.add_mutually_exclusive_group(required=True)
varsarg.add_argument("--vars", metavar="FILE", type=str,
help="File (json) containing cmor variables per EC-Earth component")
varsarg.add_argument("--drq", metavar="FILE", type=str,
help="File (json|f90 namelist|xlsx) containing cmor variables")
varsarg.add_argument("--allvars", action="store_true", default=False,
help="Read all possible variables from CMOR tables")
parser.add_argument("--tabdir", metavar="DIR", type=str, default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tabid", metavar="PREFIX", type=str, default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
args = parser.parse_args()
print()
print('Running drq2ppt with:')
print(' drq2ppt ' + cmor_utils.ScriptUtils.get_drq_vars_options(args))
print()
if args.vars is not None and not os.path.isfile(args.vars):
log.fatal("Error: Your variable list json file %s cannot be found." % args.vars)
sys.exit(' Exiting drq2ppt.')
if args.drq is not None and not os.path.isfile(args.drq):
log.fatal("Error: Your data request file %s cannot be found." % args.drq)
sys.exit(' Exiting drq2ppt.')
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default, mode=ece2cmorlib.PRESERVE, tabledir=args.tabdir,
tableprefix=args.tabid)
# Load only atmosphere variables as task targets:
try:
if getattr(args, "vars", None) is not None:
taskloader.load_tasks(args.vars, active_components=["ifs"])
elif getattr(args, "allvars", False):
taskloader.load_tasks_from_drq("allvars", active_components=["ifs"], check_prefs=False)
else:
taskloader.load_tasks_from_drq(args.drq, active_components=["ifs"], check_prefs=False)
except taskloader.SwapDrqAndVarListException as e:
log.error(e.message)
opt1, opt2 = "vars" if e.reverse else "drq", "drq" if e.reverse else "vars"
log.error("It seems you are using the --%s option where you should use the --%s option for this file"
% (opt1, opt2))
sys.exit(' Exiting drq2ppt.')
# Write the IFS input files
write_ppt_files(ece2cmorlib.tasks)
# Finishing up
ece2cmorlib.finalize_without_cmor()
def test_load_avars_drq():
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq({
"3hr": ["clt", "uas", "vas"],
"Amon": ["vas", "tas"]
})
assert len(ece2cmorlib.tasks) == 5
assert len([
t.source.get_grib_code().var_id for t in ece2cmorlib.tasks
if t.target.variable == "vas"
]) == 2
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_avars_drq_json():
ece2cmorlib.initialize_without_cmor()
try:
avars = {"3hr": ["clt", "uas", "vas"], "Amon": ["vas", "tas"]}
taskloader_test.setup_drq(avars)
taskloader.load_tasks_from_drq(drqpath)
eq_(len(ece2cmorlib.tasks), 5)
eq_(
2,
len([
t.source.get_grib_code().var_id for t in ece2cmorlib.tasks
if t.target.variable == "vas"
]))
finally:
taskloader_test.cleanup_drq()
ece2cmorlib.finalize_without_cmor()
def test_load_oavars_ocean():
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq(
{
"3hr": ["clt", "uas"],
"Amon": ["vas", "tas"],
"Omon": ["tossq"]
},
active_components=["nemo"])
assert len(ece2cmorlib.tasks) == 1
assert len([
t for t in ece2cmorlib.tasks
if isinstance(t.source, cmor_source.netcdf_source)
]) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_oavars():
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq({
"3hr": ["clt", "uas"],
"Amon": ["vas", "tas"],
"Omon": ["tossq"]
})
assert len(ece2cmorlib.tasks) == 5
assert len([
t for t in ece2cmorlib.tasks
if isinstance(t.source, cmor_source.ifs_source)
]) == 4
assert len([
t for t in ece2cmorlib.tasks
if isinstance(t.source, cmor_source.netcdf_source)
]) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_skip_alevel():
def ifs_model_level_variable(target):
zaxis, levs = cmor_target.get_z_axis(target)
return zaxis not in ["alevel", "alevhalf"]
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq(
{
"3hr": ["clt", "uas", "vas"],
"CFmon": ["clwc", "hur", "ps"]
},
target_filters={"model level": ifs_model_level_variable})
assert len(ece2cmorlib.tasks) == 4
assert len([
t for t in ece2cmorlib.tasks if t.target.table == "CFmon"
]) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_unit_conv():
ece2cmorlib.initialize_without_cmor()
try:
taskloader.load_tasks_from_drq({"Amon": ["prc", "rsus", "zg"]})
assert len(ece2cmorlib.tasks) == 3
prctask = [
t for t in ece2cmorlib.tasks if t.target.variable == "prc"
][0]
rsustask = [
t for t in ece2cmorlib.tasks if t.target.variable == "rsus"
][0]
zgtask = [
t for t in ece2cmorlib.tasks if t.target.variable == "zg"
][0]
assert getattr(prctask, cmor_task.conversion_key) == "vol2flux"
assert getattr(rsustask, cmor_task.conversion_key) == "cum2inst"
assert getattr(zgtask, cmor_task.conversion_key) == "pot2alt"
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_tsl_table_override():
ece2cmorlib.initialize_without_cmor()
try:
tasks = taskloader.load_tasks_from_drq({
"6hrPlevPt": ["tsl"],
"Lmon": ["tsl"]
})
eq_(len(tasks), 2)
for t in tasks:
if t.target.table == "6hrPlevPt":
ok_(not hasattr(t.source, "expr"))
ok_(t.source.get_grib_code() == cmor_source.grib_code(139))
else:
ok_("merge" in getattr(t.source, "expr"))
finally:
ece2cmorlib.finalize_without_cmor()
def main():
parser = argparse.ArgumentParser(
description=
"Create NEMO XIOS file_def input files for a given CMIP6 data request")
varsarg = parser.add_mutually_exclusive_group(required=True)
varsarg.add_argument(
"--vars",
metavar="FILE",
type=str,
help="File (json) containing cmor variables per EC-Earth component")
varsarg.add_argument(
"--drq",
metavar="FILE",
type=str,
help="File (json|f90 namelist|xlsx) containing cmor variables")
parser.add_argument("--tabdir",
metavar="DIR",
type=str,
default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tabid",
metavar="PREFIX",
type=str,
default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
args = parser.parse_args()
print ""
print "Running drq2ppt.py with:"
print "./drq2file_def-nemo.py " + cmor_utils.ScriptUtils.get_drq_vars_options(
args)
print ""
if args.vars is not None and not os.path.isfile(args.vars):
log.fatal("Your variable list json file %s cannot be found." %
args.vars)
sys.exit(' Exiting drq2file_def-nemo.')
if args.drq is not None and not os.path.isfile(args.drq):
log.fatal("Your data request file %s cannot be found." % args.drq)
sys.exit(' Exiting drq2file_def-nemo.')
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default,
mode=ece2cmorlib.PRESERVE,
tabledir=args.tabdir,
tableprefix=args.tabid)
# Load only ocean variables as task targets:
try:
if getattr(args, "vars", None) is not None:
taskloader.load_tasks(args.vars, active_components=["nemo"])
else:
taskloader.load_tasks_from_drq(args.drq,
active_components=["nemo"],
check_prefs=False)
except taskloader.SwapDrqAndVarListException as e:
log.error(e.message)
opt1, opt2 = "vars" if e.reverse else "drq", "drq" if e.reverse else "vars"
log.error(
"It seems you are using the --%s option where you should use the --%s option for this file"
% (opt1, opt2))
sys.exit(' Exiting drq2file_def-nemo.')
for task in ece2cmorlib.tasks:
print ' {:15} {:9} {:15} {}'.format(task.target.variable,
task.target.table,
task.target.units,
task.target.frequency)
#print task.target.__dict__
print ' Number of activated data request tasks is', len(ece2cmorlib.tasks)
# READING THE BASIC FILE_DEF FILE:
if os.path.isfile(basic_file_def_file_name) == False:
print ' The file ', basic_file_def_file_name, ' does not exist.'
sys.exit(' stop')
tree_basic_file_def = xmltree.parse(basic_file_def_file_name)
root_basic_file_def = tree_basic_file_def.getroot(
) # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
#field_elements_basic_file_def = root_basic_file_def[0][:]
total_layer_equivalent = 0
count = 0
for field in root_basic_file_def.findall('.//field[@id]'):
for task in ece2cmorlib.tasks:
if field.attrib["name"] == task.target.variable and field.attrib[
"table"] == task.target.table:
field.attrib["enabled"] = "True"
count = count + 1
#print field.attrib["name"], field.attrib["table"]
# NEMO Volume estimate: estimate the number of 2D layers per variable in output due to the number of time steps per year:
if task.target.frequency == 'yr':
layer_number_due_to_freq = 1
elif task.target.frequency == 'mon':
layer_number_due_to_freq = 12
elif task.target.frequency == 'day':
layer_number_due_to_freq = 365
elif task.target.frequency == '3hrPt':
layer_number_due_to_freq = 365.25 * 8.
elif task.target.frequency == 'fx':
layer_number_due_to_freq = 0
else:
print '\n Unknown frequency in NEMO Volume estimate for: {:15} at table: {:9} with frequency: {}\n'.format(
task.target.variable, task.target.table,
task.target.frequency)
layer_number_due_to_freq = 0
# NEMO Volume estimate: estimate the number vertical layers per variable:
zdim = getattr(task.target, "z_dims", [])
if len(zdim) == 0:
vertical_dim = 1
else:
if zdim[0] == 'olevel':
vertical_dim = 75
#elif zdim[0] == 'typesea':
#elif zdim[0] == 'depth100m':
#elif zdim[0] == 'depth0m':
else:
vertical_dim = 1
# NEMO Volume estimate: calculate the number of 2D layers in output due to the number of time steps & the number of vertical layers per year per variable:
layers_per_var_per_yr = layer_number_due_to_freq * vertical_dim
# NEMO Volume estimate: and for all variables together:
total_layer_equivalent = total_layer_equivalent + layers_per_var_per_yr
#print(' {:3} varname: {:15} freq: {:5} table: {:7} zdim: {:30} vertical dim: {:3} {:2} {:8} layers per var per yr: {:8}'.format(count, task.target.variable, task.target.frequency, task.target.table, getattr(task.target, "z_dims", []), vertical_dim, len(zdim), layer_number_due_to_freq, layers_per_var_per_yr ))
# After the table attribute has been used to match with the data request, the table attribute is removed here because it is not a valid XIOS attribute:
field.attrib.pop('table', None)
# Write the NEMO XIOS file_def input files:
tree_basic_file_def.write(file_def_file_name)
print '\n With a 2D layer equivalent of ', total_layer_equivalent, ' the NEMO Volume estimate for this CMIP6 data request is ', total_layer_equivalent * 0.43 / 1000.0, ' GB per year\n'
print ' The number of variables which is enabled in', file_def_file_name, ' is', count
volume_estimate = open('volume-estimate-nemo.txt', 'w')
volume_estimate.write(
' \nEC-Earth3 NEMO volume estimates of generated output:{}'.format(
'\n'))
volume_estimate.write(
' Volume estimate for the ORCA1L75 grid: {} GB/yr{}'.format(
total_layer_equivalent * 0.43 / 1000.0, '\n'))
volume_estimate.write(
' Volume estimate for the ORCA025L75 grid: {} GB/yr{}'.format(
total_layer_equivalent * 5.76 / 1000.0, '\n'))
volume_estimate.write(
' With {:8} horizontal data slices per year across the vertical and time dimension.{}'
.format(int(total_layer_equivalent), '\n\n'))
volume_estimate.close()
# SPLIT THE FILE_DEF FILE IN THREE FILE_DEF FILES FOR OPA, LIM AND PISCES:
# FILE_DEF FILE FOR OPA:
tree_opa = xmltree.parse(file_def_file_name)
root_opa = tree_opa.getroot(
) # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
for file_element in root_opa.findall('./file_group/file'):
# Get the model component info from this attribute by using a regular expression:
model_component = re.search(
'_(.+?)_', file_element.attrib["name_suffix"]).group(1)
if model_component == 'lim' or model_component == 'pisces':
#print ' Remove file for opa file_def:', file_element.attrib["id"], model_component
# Remove this file element from its parent element the file_group element:
root_opa[0].remove(file_element)
root_opa[0].attrib["id"] = "id_file_group_opa"
tree_opa.write(file_def_opa_file_name)
# FILE_DEF FILE FOR LIM:
tree_lim = xmltree.parse(file_def_file_name)
root_lim = tree_lim.getroot(
) # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
for file_element in root_lim.findall('./file_group/file'):
# Get the model component info from this attribute by using a regular expression:
model_component = re.search(
'_(.+?)_', file_element.attrib["name_suffix"]).group(1)
if model_component == 'opa' or model_component == 'pisces':
#print ' Remove file for lim file_def:', file_element.attrib["id"], model_component
# Remove this file element from its parent element the file_group element:
root_lim[0].remove(file_element)
root_lim[0].attrib["id"] = "id_file_group_lim"
tree_lim.write(file_def_lim_file_name)
# FILE_DEF FILE FOR PISCES:
tree_pisces = xmltree.parse(file_def_file_name)
root_pisces = tree_pisces.getroot(
) # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
for file_element in root_pisces.findall('./file_group/file'):
# Get the model component info from this attribute by using a regular expression:
model_component = re.search(
'_(.+?)_', file_element.attrib["name_suffix"]).group(1)
if model_component == 'opa' or model_component == 'lim':
#print ' Remove file for pisces file_def:', file_element.attrib["id"], model_component
# Remove this file element from its parent element the file_group element:
root_pisces[0].remove(file_element)
root_pisces[0].attrib["id"] = "id_file_group_pisces"
tree_pisces.write(file_def_pisces_file_name)
# Finishing up
ece2cmorlib.finalize_without_cmor()
# PRODUCE FILE_DEF FILES FOR OPA, LIM AND PISCES WITH ONLY ENABLED VARIABLES:
# FILE_DEF FILE FOR OPA WITH ONLY ENABLED VARIABLES:
tree_opa_enabled_only = xmltree.parse(file_def_opa_file_name)
root_opa_enabled_only = tree_opa_enabled_only.getroot(
) # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
for file_element in root_opa_enabled_only.findall('./file_group/file'):
for field_element in file_element.findall('field[@enabled="False"]'):
file_element.remove(field_element)
tree_opa_enabled_only.write(file_def_opa_file_name_compact)
# FILE_DEF FILE FOR LIM WITH ONLY ENABLED VARIABLES:
tree_lim_enabled_only = xmltree.parse(file_def_lim_file_name)
root_lim_enabled_only = tree_lim_enabled_only.getroot(
) # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
for file_element in root_lim_enabled_only.findall('./file_group/file'):
for field_element in file_element.findall('field[@enabled="False"]'):
file_element.remove(field_element)
tree_lim_enabled_only.write(file_def_lim_file_name_compact)
# FILE_DEF FILE FOR PISCES WITH ONLY ENABLED VARIABLES:
tree_pisces_enabled_only = xmltree.parse(file_def_pisces_file_name)
root_pisces_enabled_only = tree_pisces_enabled_only.getroot(
) # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements
for file_element in root_pisces_enabled_only.findall('./file_group/file'):
for field_element in file_element.findall('field[@enabled="False"]'):
file_element.remove(field_element)
tree_pisces_enabled_only.write(file_def_pisces_file_name_compact)
def main(args=None):
if args is None:
pass
formatter = lambda prog: argparse.ArgumentDefaultsHelpFormatter(
prog, max_help_position=30)
parser = argparse.ArgumentParser(
description="Post-processing and cmorization of EC-Earth output",
formatter_class=formatter)
required = parser.add_argument_group("required arguments")
parser.add_argument(
"datadir",
metavar="DIR",
type=str,
help="EC-Earth data directory, i.e. for a given component, "
"for a given leg")
parser.add_argument("--exp",
metavar="EXPID",
type=str,
default="ECE3",
help="Experiment prefix")
varsarg = required.add_mutually_exclusive_group(required=True)
varsarg.add_argument(
"--varlist",
metavar="FILE",
type=str,
help=
"File (json) containing cmor variables grouped per table, grouped per EC-Earth component"
)
varsarg.add_argument(
"--drq",
metavar="FILE",
type=str,
help=
"File (json|f90 namelist|xlsx) containing cmor variables, grouped per table"
)
required.add_argument("--meta",
metavar="FILE.json",
type=str,
required=True,
help="Input metadata file")
parser.add_argument("--odir",
metavar="DIR",
type=str,
default=None,
help="Output directory, by default the "
"metadata \'outpath\' entry")
cmor_utils.ScriptUtils.add_model_exclusive_options(parser, "ece2cmor")
parser.add_argument(
"--ececonf",
metavar='|'.join(components.ece_configs.keys()),
type=str,
help="EC-Earth configuration (only used with --drq option)")
parser.add_argument("--refd",
metavar="YYYY-mm-dd",
type=str,
default="1850-01-01",
help="Reference date for output time axes")
parser.add_argument("--npp",
metavar="N",
type=int,
default=8,
help="Number of parallel tasks (only relevant for "
"IFS cmorization")
parser.add_argument("--log",
action="store_true",
default=False,
help="Write to log file")
parser.add_argument("--flatdir",
action="store_true",
default=False,
help="Do not create sub-directories in "
"output folder")
parser.add_argument("--tabledir",
metavar="DIR",
type=str,
default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tableprefix",
metavar="PREFIX",
type=str,
default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
parser.add_argument("--tmpdir",
metavar="DIR",
type=str,
default="/tmp/ece2cmor",
help="Temporary working directory")
parser.add_argument(
"--overwritemode",
metavar="MODE",
type=str,
default="preserve",
help="MODE:preserve|replace|append, CMOR netcdf overwrite mode",
choices=["preserve", "replace", "append"])
parser.add_argument("--skip_alevel_vars",
action="store_true",
default=False,
help="Prevent loading atmospheric "
"model-level variables")
parser.add_argument(
"-V",
"--version",
action="version",
version="%(prog)s {version}".format(version=__version__.version))
# Deprecated arguments, only for backward compatibility
parser.add_argument("--ncdo",
metavar="N",
type=int,
default=4,
help=argparse.SUPPRESS)
parser.add_argument("--nomask",
action="store_true",
default=False,
help=argparse.SUPPRESS)
parser.add_argument("--nofilter",
action="store_true",
default=False,
help=argparse.SUPPRESS)
parser.add_argument("--atm",
action="store_true",
default=False,
help="Deprecated! Use --ifs instead")
parser.add_argument("--oce",
action="store_true",
default=False,
help="Deprecated! Use --nemo instead")
parser.add_argument("--conf",
action="store_true",
help="Deprecated! Use --meta instead")
parser.add_argument("--vars",
action="store_true",
help="Deprecated! Use --varlist instead")
cmor_utils.ScriptUtils.add_model_tabfile_options(parser)
args = parser.parse_args()
cmor_utils.ScriptUtils.set_custom_tabfiles(args)
logfile = None
logformat = "%(asctime)s %(levelname)s:%(name)s: %(message)s"
logdateformat = "%Y-%m-%d %H:%M:%S"
if getattr(args, "log", False):
dirs = os.path.abspath(args.datadir).split(os.sep)
fname = '-'.join([args.exp] + dirs[-2:] +
[time.strftime("%Y%m%d%H%M%S", time.gmtime())])
logfile = '.'.join([fname, "log"])
logging.basicConfig(filename=logfile,
level=logging.DEBUG,
format=logformat,
datefmt=logdateformat)
else:
logging.basicConfig(level=logging.DEBUG,
format=logformat,
datefmt=logdateformat)
if not os.path.isdir(args.datadir):
log.fatal("Your data directory argument %s cannot be found." %
args.datadir)
sys.exit(' Exiting ece2cmor.')
if args.varlist is not None and not os.path.isfile(args.varlist):
log.fatal("Your variable list json file %s cannot be found." %
args.varlist)
sys.exit(' Exiting ece2cmor.')
if args.drq is not None and not os.path.isfile(args.drq):
log.fatal("Your data request file %s cannot be found." % args.drq)
sys.exit(' Exiting ece2cmor.')
if not os.path.isfile(args.meta):
log.fatal("Your metadata file %s cannot be found." % args.meta)
sys.exit(' Exiting ece2cmor.')
modedict = {
"preserve": ece2cmorlib.PRESERVE,
"append": ece2cmorlib.APPEND,
"replace": ece2cmorlib.REPLACE
}
# Initialize ece2cmor:
ece2cmorlib.initialize(args.meta,
mode=modedict[args.overwritemode],
tabledir=args.tabledir,
tableprefix=args.tableprefix,
outputdir=args.odir,
logfile=logfile,
create_subdirs=(not args.flatdir))
ece2cmorlib.enable_masks = not args.nomask
ece2cmorlib.auto_filter = not args.nofilter
active_components = cmor_utils.ScriptUtils.get_active_components(
args, args.ececonf)
filters = None
if args.skip_alevel_vars:
def ifs_model_level_variable(target):
zaxis, levs = cmor_target.get_z_axis(target)
return zaxis not in ["alevel", "alevhalf"]
filters = {"model level": ifs_model_level_variable}
try:
if getattr(args, "varlist", None) is not None:
taskloader.load_tasks(args.varlist,
active_components=active_components,
target_filters=filters,
check_duplicates=True)
else:
taskloader.load_tasks_from_drq(args.drq,
active_components=["ifs"],
target_filters=filters,
check_prefs=True)
except taskloader.SwapDrqAndVarListException as e:
log.error(e.message)
opt1, opt2 = "vars" if e.reverse else "drq", "drq" if e.reverse else "vars"
log.error(
"It seems you are using the --%s option where you should use the --%s option for this file"
% (opt1, opt2))
sys.exit(' Exiting ece2cmor.')
refdate = datetime.datetime.combine(dateutil.parser.parse(args.refd),
datetime.datetime.min.time())
if "ifs" in active_components:
ece2cmorlib.perform_ifs_tasks(args.datadir,
args.exp,
refdate=refdate,
tempdir=args.tmpdir,
taskthreads=args.npp,
cdothreads=args.ncdo)
if "nemo" in active_components:
ece2cmorlib.perform_nemo_tasks(args.datadir, args.exp, refdate)
if "lpjg" in active_components:
ece2cmorlib.perform_lpjg_tasks(args.datadir, args.tmpdir, args.exp,
refdate)
if "tm5" in active_components:
ece2cmorlib.perform_tm5_tasks(args.datadir, args.tmpdir, args.exp,
refdate)
# if procNEWCOMPONENT in active_components:
# ece2cmorlib.perform_NEWCOMPONENT_tasks(args.datadir, args.exp, refdate)
ece2cmorlib.finalize()
def main():
parser = argparse.ArgumentParser(
description=
"Estimates the volume of the output from TM5 for a given CMIP6 data "
"request for EC-Earth3")
varsarg = parser.add_mutually_exclusive_group(required=True)
varsarg.add_argument(
"--vars",
metavar="FILE",
type=str,
help="File (json) containing cmor variables per EC-Earth component")
varsarg.add_argument(
"--drq",
metavar="FILE",
type=str,
help="File (json|f90 namelist|xlsx) containing cmor variables")
parser.add_argument("--tabdir",
metavar="DIR",
type=str,
default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tabid",
metavar="PREFIX",
type=str,
default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
parser.add_argument("--short",
action="store_true",
default=False,
help="Leave out the tasklist")
args = parser.parse_args()
print()
print('Running estimate_tm5_volume.py with:')
print(' estimate_tm5_volume ' +
cmor_utils.ScriptUtils.get_drq_vars_options(args))
print()
if args.vars is not None and not os.path.isfile(args.vars):
log.fatal("Error: Your variable list json file %s cannot be found." %
args.vars)
sys.exit(' Exiting estimate_tm5_volume.')
if args.drq is not None and not os.path.isfile(args.drq):
log.fatal("Error: Your data request file %s cannot be found." %
args.drq)
sys.exit(' Exiting estimate_tm5_volume.')
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default,
mode=ece2cmorlib.PRESERVE,
tabledir=args.tabdir,
tableprefix=args.tabid)
# Load only TM5 variables as task targets:
try:
if getattr(args, "vars", None) is not None:
taskloader.load_tasks(args.vars, active_components=["tm5"])
else:
taskloader.load_tasks_from_drq(args.drq,
active_components=["tm5"],
check_prefs=False)
except taskloader.SwapDrqAndVarListException as e:
log.error(e.message)
opt1, opt2 = "vars" if e.reverse else "drq", "drq" if e.reverse else "vars"
log.error(
"It seems you are using the --%s option where you should use the --%s option for this file"
% (opt1, opt2))
sys.exit(' Exiting estimate_tm5_volume.')
for task in ece2cmorlib.tasks:
print(' {:15} {:9} {:15} {}'.format(task.target.variable,
task.target.table,
task.target.units,
task.target.frequency))
#print(task.target.__dict__)
print(' Number of activated data request tasks is', len(ece2cmorlib.tasks))
total_layer_equivalent = 0
count = 0
per_freq = {}
task_per_freq = {}
for task in ece2cmorlib.tasks:
count = count + 1
if not getattr(args, 'short', False):
print(' {:15} {:9} {:15} {}'.format(task.target.variable,
task.target.table,
task.target.units,
task.target.frequency))
if task.target.table not in task_per_freq.keys():
task_per_freq[task.target.table] = 0
if task.target.table not in per_freq.keys():
per_freq[task.target.table] = 0
# TM5 Volume estimate: estimate the number of 2D layers per variable in output due to the number of time steps per year:
if task.target.frequency == 'yr':
layer_number_due_to_freq = 1.
elif task.target.frequency == 'mon':
layer_number_due_to_freq = 12.
elif task.target.frequency == 'day':
layer_number_due_to_freq = 365.
elif task.target.frequency == '3hrPt':
layer_number_due_to_freq = 0.
elif task.target.frequency == '6hrPt':
layer_number_due_to_freq = 365.25 * 4.
elif task.target.frequency == '1hr':
layer_number_due_to_freq = 365.25 * 24.
elif task.target.frequency == 'fx':
layer_number_due_to_freq = 0.
elif task.target.frequency == 'monC':
layer_number_due_to_freq = 1.0 / 30.0 # Number of climate points: 1 per 30 year?
elif task.target.frequency == 'subhrPt':
#layer_number_due_to_freq = 365.25 * 12. # At least hourly, thus sofar under limit (Actually there should't be (sub) houry variables available?).
layer_number_due_to_freq = 0. # Because there won't be any subhourly output from TM5.
else:
print(
'\n Unknown frequency in TM5 Volume estimate for: {:15} at table: {:9} with frequency: {}\n'
.format(task.target.variable, task.target.table,
task.target.frequency))
layer_number_due_to_freq = 0.
if task.target.table == 'Emon':
layer_number_due_to_freq = 0.
# TM5 Volume estimate: estimate the number vertical layers per variable:
zdim = getattr(task.target, "z_dims", [])
if len(zdim) == 0:
vertical_dim = 1.
else:
if zdim[0] == 'alevel':
vertical_dim = 35. #34 + 1 for ps
elif zdim[0] == 'alevhalf':
vertical_dim = 36. # 34 + 1 for ps
elif zdim[0] == 'plev39':
vertical_dim = 39.
elif zdim[0] == 'plev19':
vertical_dim = 19.
else:
vertical_dim = 1.
task_per_freq[task.target.table] = task_per_freq[task.target.table] + 1
# TM5 Volume estimate: calculate the number of 2D layers in output due to the number of time steps & the number of vertical layers per year per variable:
layers_per_var_per_yr = layer_number_due_to_freq * vertical_dim
if task.target.table == 'AERmonZ':
per_freq[task.target.table] = per_freq[
task.target.table] + layers_per_var_per_yr / 120.0
# TM5 Volume estimate: and for all variables together:
total_layer_equivalent = total_layer_equivalent + layers_per_var_per_yr / 120.0
else:
per_freq[task.target.table] = per_freq[
task.target.table] + layers_per_var_per_yr
# TM5 Volume estimate: and for all variables together:
total_layer_equivalent = total_layer_equivalent + layers_per_var_per_yr
#print(' {:3} varname: {:15} freq: {:5} table: {:7} zdim: {:30} vertical dim: {:3} {:2} {:8} layers per var per yr: {:8}'.format(count, task.target.variable, task.target.frequency, task.target.table, getattr(task.target, "z_dims", []), vertical_dim, len(zdim), layer_number_due_to_freq, layers_per_var_per_yr ))
print('\n With a 2D layer equivalent of ', total_layer_equivalent,
' the TM5 Volume estimate for this CMIP6 data request is ',
total_layer_equivalent * 0.04 / 1000.0, ' GB per year\n')
print(
' The number of variables which is available from TM5 in EC-Erth3 for this experiment is',
count)
for i in per_freq:
#if i =='AERmonZ':
# print('Table: {} \tsize: {} GB/yr'.format(i,per_freq[i]*0.04/1000/120))
#else:
print('Table: {} \t tasks {} \tsize: {} GB/yr'.format(
i, task_per_freq[i], per_freq[i] * 0.04 / 1024.0))
#volume_estimate = open('volume-estimate-tm5.txt','w')
#volume_estimate.write(' \nEC-Earth3 TM5 Volume estimates of generated output:{}'.format('\n'))
#volume_estimate.write(' Volume estimate for the TM5 3x2 degrees grid: {} GB/yr{}'.format(total_layer_equivalent * 0.04 / 1000.0, '\n'))
#volume_estimate.write(' With {:8} horizontal data slices per year across the vertical and time dimension.{}'.format(int(total_layer_equivalent), '\n\n'))
#volume_estimate.close()
hf = 1.0 # TM5 heuristic factor
volume_estimate = open('volume-estimate-tm5.txt', 'w')
volume_estimate.write(
'Heuristic volume estimate for the raw EC-Earth3 TM5 output on the TM5 3x2 deg grid: {:6} GB per year{}'
.format(round((total_layer_equivalent * 0.04 / 1000.0) / hf, 1), '\n'))
volume_estimate.close()
# Finishing up
ece2cmorlib.finalize_without_cmor()
def main():
parser = argparse.ArgumentParser(
description=
"Estimates the volume of the output from LPJ-GUESS for a given CMIP6 "
"data request for EC-Earth3")
varsarg = parser.add_mutually_exclusive_group(required=True)
varsarg.add_argument(
"--vars",
metavar="FILE",
type=str,
help="File (json) containing cmor variables per EC-Earth component")
varsarg.add_argument(
"--drq",
metavar="FILE",
type=str,
help="File (json|f90 namelist|xlsx) containing cmor variables")
parser.add_argument("--tabdir",
metavar="DIR",
type=str,
default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tabid",
metavar="PREFIX",
type=str,
default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
args = parser.parse_args()
print ""
print "Running drq2ins.py with:"
print "./drq2ins.py " + cmor_utils.ScriptUtils.get_drq_vars_options(args)
print ""
if args.vars is not None and not os.path.isfile(args.vars):
log.fatal("Your variable list json file %s cannot be found." %
args.vars)
sys.exit(' Exiting drq2ins.')
if args.drq is not None and not os.path.isfile(args.drq):
log.fatal("Your data request file %s cannot be found." % args.drq)
sys.exit(' Exiting drq2ins.')
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default,
mode=ece2cmorlib.PRESERVE,
tabledir=args.tabdir,
tableprefix=args.tabid)
# Load only LPJ-GUESS variables as task targets:
try:
if getattr(args, "vars", None) is not None:
taskloader.load_tasks(args.vars, active_components=["lpjg"])
else:
taskloader.load_tasks_from_drq(args.drq,
active_components=["lpjg"],
check_prefs=False)
# Here we load extra permanent tasks for LPJ-GUESS because the LPJ_GUESS community likes to output these variables at any time independent wheter they are requested by the data request:
taskloader.load_tasks_from_drq(os.path.join(
os.path.dirname(__file__), "..", "resources",
"permanent-tasks.json"),
active_components=["lpjg"],
check_prefs=False)
except taskloader.SwapDrqAndVarListException as e:
log.error(e.message)
opt1, opt2 = "vars" if e.reverse else "drq", "drq" if e.reverse else "vars"
log.error(
"It seems you are using the --%s option where you should use the --%s option for this file"
% (opt1, opt2))
sys.exit(' Exiting drq2ins.')
print '\n Number of activated data request tasks is', len(
ece2cmorlib.tasks), '\n'
instruction_file = open('lpjg_cmip6_output.ins', 'w')
total_layer_equivalent = 0
count = 0
for task in ece2cmorlib.tasks:
count = count + 1
print ' {:15} {:9} {:15} {}'.format(task.target.variable,
task.target.table,
task.target.units,
task.target.frequency)
if task.target.frequency == 'yr':
instruction_file.write('file_{}_yearly "{}_yearly.out"{}'.format(
task.target.variable, task.target.variable, '\n'))
elif task.target.frequency == 'mon':
instruction_file.write('file_{}_monthly "{}_monthly.out"{}'.format(
task.target.variable, task.target.variable, '\n'))
elif task.target.frequency == 'day':
instruction_file.write('file_{}_daily "{}_daily.out"{}'.format(
task.target.variable, task.target.variable, '\n'))
elif task.target.frequency == '3hr':
print ' LPJ-GUESS does not provide three hourly (3hr) output: ', task.target.variable, task.target.table, task.target.frequency
elif task.target.frequency == '6hr':
print ' LPJ-GUESS does not provide six hourly (6hr) output: ', task.target.variable, task.target.table, task.target.frequency
elif task.target.frequency == 'yrPt':
print ' LPJ-GUESS does not provide yearly instantaneous (yrPt) output: ', task.target.variable, task.target.table, task.target.frequency
# instruction_file.write('file_{}_yearly "{}_yearly.out"{}'.format(task.target.variable, task.target.variable, '\n'))
elif task.target.frequency == '3hrPt':
print ' LPJ-GUESS does not provide three hourly instantaneous (3hrPt) output: ', task.target.variable, task.target.table, task.target.frequency
elif task.target.frequency == 'fx':
#print ' LPJ-GUESS does not provide fx output', task.target.variable, task.target.table, task.target.frequency
instruction_file.write('file_{}_once "{}_once.out"{}'.format(
task.target.variable, task.target.variable, '\n'))
elif task.target.frequency == 'monC':
#print ' LPJ-GUESS does not provide monC output', task.target.variable, task.target.table, task.target.frequency
instruction_file.write(
'file_{}_monthly_clim "{}_monthly_clim.out"{}'.format(
task.target.variable, task.target.variable, '\n'))
elif task.target.frequency == 'subhrPt':
print ' LPJ-GUESS does not provide subhourly instantaneous (subhrPt) output: ', task.target.variable, task.target.table, task.target.frequency
else:
print '\n Unknown frequency in creating the LPJG instruction file for: {:15} at table: {:9} with frequency: {}\n'.format(
task.target.variable, task.target.table, task.target.frequency)
# LPJ-GUESS Volume estimate: estimate the number of 2D layers per variable in output due to the number of time steps per year:
if task.target.frequency == 'yr':
layer_number_due_to_freq = 1
elif task.target.frequency == 'mon':
layer_number_due_to_freq = 12
elif task.target.frequency == 'day':
layer_number_due_to_freq = 365
elif task.target.frequency == '3hr':
layer_number_due_to_freq = 0 # LPJ-GUESS does not provide three hourly (3hr) output
elif task.target.frequency == '6hr':
layer_number_due_to_freq = 0 # LPJ-GUESS does not provide six hourly (6hr) output
elif task.target.frequency == 'yrPt':
layer_number_due_to_freq = 0 # LPJ-GUESS does not provide yearly instantaneous (yrPt) output
# layer_number_due_to_freq = 1
elif task.target.frequency == '3hrPt':
layer_number_due_to_freq = 365.25 * 8.
elif task.target.frequency == 'fx':
layer_number_due_to_freq = 0
elif task.target.frequency == 'monC':
layer_number_due_to_freq = 1. / 30. # Number of climate points: 1 per 30 year?
elif task.target.frequency == 'subhrPt':
#layer_number_due_to_freq = 365.25 * 12. # At least hourly, thus sofar under limit (Actually there should't be (sub) houry variables available?).
layer_number_due_to_freq = 0 # Because there won't be any subhourly output from LPJ-GUESS.
else:
print '\n Unknown frequency in LPJG Volume estimate for: {:15} at table: {:9} with frequency: {}\n'.format(
task.target.variable, task.target.table, task.target.frequency)
layer_number_due_to_freq = 0
# LPJ-GUESS Volume estimate: estimate the number vertical layers per variable:
zdim = getattr(task.target, "z_dims", [])
if len(zdim) == 0:
vertical_dim = 1
else:
if zdim[0] == 'vegtype':
vertical_dim = 31
elif zdim[0] == 'landuse':
vertical_dim = 4
elif zdim[0] == 'sdepth':
vertical_dim = 2
else:
vertical_dim = 1
# LPJ-GUESS Volume estimate: calculate the number of 2D layers in output due to the number of time steps & the number of vertical layers per year per variable:
layers_per_var_per_yr = layer_number_due_to_freq * vertical_dim
# LPJ-GUESS Volume estimate: and for all variables together:
total_layer_equivalent = total_layer_equivalent + layers_per_var_per_yr
#print(' {:3} varname: {:15} freq: {:5} table: {:7} zdim: {:30} vertical dim: {:3} {:2} {:8} layers per var per yr: {:8}'.format(count, task.target.variable, task.target.frequency, task.target.table, getattr(task.target, "z_dims", []), vertical_dim, len(zdim), layer_number_due_to_freq, layers_per_var_per_yr ))
instruction_file.close()
print '\n With a 2D layer equivalent of ', total_layer_equivalent, ' the LPJ-GUESS Volume estimate for this CMIP6 data request at T255 grid is ', total_layer_equivalent * 0.12 / 1000.0, ' GB per year\n'
print ' The number of variables which is available from LPJ-GUESS in EC-Erth3 for this experiment is', count
volume_estimate = open('volume-estimate-lpj-guess.txt', 'w')
volume_estimate.write(
' \nEC-Earth3 LPJ-GUESS Volume estimates of generated output:{}'.
format('\n'))
volume_estimate.write(
' Volume estimate for the LPJ-GUESS T255 grid: {} GB/yr{}'.format(
total_layer_equivalent * 0.12 / 1000.0, '\n'))
volume_estimate.write(
' With {:8} horizontal data slices per year across the vertical and time dimension.{}'
.format(int(total_layer_equivalent), '\n\n'))
volume_estimate.close()
volume_estimate.close()
# Finishing up
ece2cmorlib.finalize_without_cmor()
# See #546: Add the variable fVegOther which is not part of the data request and has no cmor name, to the LPJ-GUESS instruction file:
with open("lpjg_cmip6_output.ins", "a") as instruction_file:
instruction_file.write(
'file_fVegOther_monthly "fVegOther_monthly.out"\n')
instruction_file.close()
