def test_expr_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon")
src = cmor_source.ifs_source.read("214.128",
"sqrt(sqr(var165)+sqr(var166))")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk])
filepath = os.path.join(tmp_path, "166.128.105_165.128.105.3")
assert os.path.isfile(filepath)
assert getattr(tsk, cmor_task.filter_output_key) == [filepath]
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
assert param in [165, 166]
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
assert newdate == date + 1
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
assert newtime == (time + 300) % 2400
time = newtime
os.remove(filepath)
def test_surf_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "79.128.1")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, cmor_task.output_path_key), filepath)
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
eq_(param, 79)
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
eq_(newtime, (time + 300) % 2400)
time = newtime
os.remove(filepath)
def main(args):
parser = optparse.OptionParser()
parser.add_option("-c",
"--conf",
dest="conf",
help="CMOR3 meta data json file path",
metavar="FILE",
default=ece2cmorlib.conf_path_default)
parser.add_option("-d",
"--dir",
dest="dir",
help="IFS output directory",
default=datadir)
parser.add_option("-e",
"--exp",
dest="exp",
help="Experiment name (prefix)",
default="exp")
(opt, args) = parser.parse_args()
# Initialize ece2cmorlib with metadata and experiment prefix:
ece2cmorlib.initialize(opt.conf)
# Load the variables as task targets:
jsonloader.load_targets(variables)
# Execute the cmorization:
ece2cmorlib.perform_nemo_tasks(opt.dir, opt.exp, startdate, interval)
def test_expr_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon")
src = cmor_source.ifs_source.read(
"var214=sqrt(sqr(var165)+sqr(var166))")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "165.128.105_166.128.105")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, "path"), filepath)
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
ok_(param in [165, 166])
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
eq_(newtime, (time + 300) % 2400)
time = newtime
os.remove(filepath)
def test_load_ovars():
ece2cmorlib.initialize()
try:
taskloader.load_targets({"Omon": ["tossq", "so", "thetao"], "Oday": ["sos"]})
eq_(len(ece2cmorlib.tasks), 4)
finally:
ece2cmorlib.finalize()
def test_load_expressions():
ece2cmorlib.initialize()
try:
taskloader.load_targets({"day": ["sfcWindmax"]})
eq_("var214=sqrt(sqr(var165)+sqr(var166))", getattr(ece2cmorlib.tasks[0].source, "expr"))
finally:
ece2cmorlib.finalize()
def test_surf_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk])
filepath = os.path.join(tmp_path, "79.128.1.3")
assert os.path.isfile(filepath)
assert getattr(tsk, cmor_task.filter_output_key) == [filepath]
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
assert param == 79
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
assert newdate == date + 1
date = newdate
newtime = reader.get_field(grib_file.time_key)
assert newtime == (time + 300) % 2400
time = newtime
os.remove(filepath)
def test_pressure_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("ua", "Amon")
src = cmor_source.ifs_source.read("131.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "131.128.210.6")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, cmor_task.filter_output_key), [filepath])
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
eq_(param, 131)
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
eq_(newtime, (time + 600) % 2400)
time = newtime
os.remove(filepath)
def test_create_task():
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
ece2cmorlib.add_task(tsk)
assert tsk in ece2cmorlib.tasks
ece2cmorlib.finalize()
def setUp(self):
self.temp_dir = "/tmp/ece2cmor3/ifs2cmor_test"
if not os.path.exists(self.temp_dir):
os.makedirs(self.temp_dir)
self.startdate = datetime.datetime(1950, 1, 1)
self.refdate = datetime.datetime(1850, 1, 1)
ece2cmorlib.initialize()
cmor.set_cur_dataset_attribute("calendar", calendar_)
def test_load_avars():
ece2cmorlib.initialize()
try:
taskloader.load_targets({"3hr": ["clt", "uas", "vas"], "Amon": ["vas", "tas"]})
eq_(len(ece2cmorlib.tasks), 5)
eq_(2, len([t.source.get_grib_code().var_id for t in ece2cmorlib.tasks if t.target.variable == "vas"]))
finally:
ece2cmorlib.finalize()
def test_load_oavars():
ece2cmorlib.initialize()
try:
taskloader.load_targets({"3hr": ["clt", "uas"], "Amon": ["vas", "tas"], "Omon": ["tossq"]})
eq_(len(ece2cmorlib.tasks), 5)
eq_(4, len([t for t in ece2cmorlib.tasks if isinstance(t.source, cmor_source.ifs_source)]))
eq_(1, len([t for t in ece2cmorlib.tasks if isinstance(t.source, cmor_source.netcdf_source)]))
finally:
ece2cmorlib.finalize()
def test_load_clt(self):
ece2cmorlib.initialize()
try:
taskloader.load_targets({"3hr": ["clt"]})
eq_(len(ece2cmorlib.tasks), 1)
src = ece2cmorlib.tasks[0].source
eq_(src.get_grib_code().var_id, 164)
finally:
ece2cmorlib.finalize()
def main(args):
parser = optparse.OptionParser()
parser.add_option("-d",
"--dir",
dest="dir",
help="IFS output directory (optional)",
default=datdir)
parser.add_option("-c",
"--conf",
dest="conf",
help="Input metadata file (optional)",
default=ece2cmorlib.conf_path_default)
parser.add_option("-e",
"--exp",
dest="exp",
help="Experiment prefix (optional)",
default="ECE3")
parser.add_option("-t",
"--tmp",
dest="temp",
help="Temporary working directory (optional)",
default=tmpdir)
parser.add_option("-v",
"--var",
dest="vars",
help="Input variable list (optional)",
default=varfile)
(opt, args) = parser.parse_args()
# Initialize ece2cmorlib with experiment prefix:
ece2cmorlib.initialize(opt.conf)
# Load the variables as task targets:
jsonloader.load_targets(opt.vars)
# Remove targets that are constructed from six-hourly data:
ece2cmorlib.tasks = [t for t in ece2cmorlib.tasks if is3hrtask(t)]
# Execute the cmorization:
if (opt.dir == datdir):
ece2cmorlib.perform_ifs_tasks(opt.dir,
opt.exp,
startdate,
interval,
outputfreq=3,
tempdir=opt.temp,
taskthreads=1)
else:
ece2cmorlib.perform_ifs_tasks(opt.dir,
opt.exp,
startdate,
interval,
outputfreq=3,
tempdir=opt.temp)
def test_load_tos_3hr():
ece2cmorlib.initialize()
try:
taskloader.load_tasks_from_drq({"3hr": ["tos"]},
active_components=["ifs"])
assert not any(ece2cmorlib.tasks)
taskloader.load_tasks_from_drq({"3hr": ["tos"]},
active_components=["nemo"])
assert len(ece2cmorlib.tasks) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_duplicate_task():
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src1 = cmor_source.ifs_source.read("49.128")
tsk1 = cmor_task.cmor_task(src1, tgt)
ece2cmorlib.add_task(tsk1)
src2 = cmor_source.ifs_source.read("79.128")
tsk2 = cmor_task.cmor_task(src2, tgt)
ece2cmorlib.add_task(tsk2)
assert len(ece2cmorlib.tasks) == 1
assert tsk2 in ece2cmorlib.tasks
ece2cmorlib.finalize()
def test_load_unit_conv():
ece2cmorlib.initialize()
try:
taskloader.load_targets({"Amon": ["prc", "rsus", "zg"]})
eq_(len(ece2cmorlib.tasks), 3)
prctask = [t for t in ece2cmorlib.tasks if t.target.variable == "prc"][0]
rsustask = [t for t in ece2cmorlib.tasks if t.target.variable == "rsus"][0]
zgtask = [t for t in ece2cmorlib.tasks if t.target.variable == "zg"][0]
eq_("vol2flux", getattr(prctask, cmor_task.conversion_key))
eq_("cum2inst", getattr(rsustask, cmor_task.conversion_key))
eq_("pot2alt", getattr(zgtask, cmor_task.conversion_key))
finally:
ece2cmorlib.finalize()
def test_validate_tasks():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt1 = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src1 = cmor_source.ifs_source.read("79.128")
tsk1 = cmor_task.cmor_task(src1, tgt1)
tgt2 = ece2cmorlib.get_cmor_target("ua", "Amon")
src2 = cmor_source.ifs_source.read("131.128")
tsk2 = cmor_task.cmor_task(src2, tgt2)
valid_tasks, varstasks = grib_filter.validate_tasks([tsk1, tsk2])
assert valid_tasks == [tsk1, tsk2]
key1 = (79, 128, grib_file.surface_level_code, 0,
cmor_source.ifs_grid.point)
key2 = (131, 128, grib_file.pressure_level_Pa_code, 92500,
cmor_source.ifs_grid.spec)
assert varstasks[key1] == [tsk1]
assert varstasks[key2] == [tsk2]
ltype, plevs = cmor_target.get_z_axis(tgt2)
levs = sorted([float(p) for p in plevs])
levcheck = sorted([k[3] for k in varstasks if k[0] == 131])
assert levs == levcheck
def main(args=None):
if args is None:
pass
varlist_path_default = os.path.join(os.path.dirname(__file__), "resources",
"varlist.json")
parser = argparse.ArgumentParser(
description="Post-processing and cmorization of EC-Earth output",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("datadir", metavar="DIR", type=str)
parser.add_argument("date", metavar="YYYY-mm-dd", type=str)
parser.add_argument(
"--vars",
metavar="FILE",
type=str,
default=varlist_path_default,
help="File (json|f90 namelist|xlsx) containing cmor variables")
parser.add_argument("--conf",
metavar="FILE.json",
type=str,
default=ece2cmorlib.conf_path_default,
help="Input metadata file")
parser.add_argument("--exp",
metavar="EXPID",
type=str,
default="ECE3",
help="Experiment prefix")
parser.add_argument(
"--refd",
metavar="YYYY-mm-dd",
type=str,
default=None,
help="Reference date (for atmosphere data), by default the start date")
parser.add_argument("--mode",
metavar="MODE",
type=str,
default="preserve",
help="CMOR netcdf mode",
choices=["preserve", "replace", "append"])
parser.add_argument(
"--freq",
metavar="N",
type=int,
default=3,
help=
"IFS output frequency, in hours (not required if autofilter is used)")
parser.add_argument("--tabdir",
metavar="DIR",
type=str,
default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tabid",
metavar="PREFIX",
type=str,
default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
parser.add_argument("--tmpdir",
metavar="DIR",
type=str,
default="/tmp/ece2cmor",
help="Temporary working directory")
parser.add_argument("--npp",
metavar="N",
type=int,
default=8,
help="Number of post-processing threads")
parser.add_argument("--tmpsize",
metavar="X",
type=float,
default=float("inf"),
help="Size of tempdir (in GB) that triggers flushing")
parser.add_argument(
"--ncdo",
metavar="N",
type=int,
default=4,
help="Number of available threads per CDO postprocessing task")
parser.add_argument("--nomask",
action="store_true",
default=False,
help="Disable masking of fields")
parser.add_argument("--filter",
action="store_true",
default=False,
help="Automatic filtering of grib files")
model_attributes, model_tabfile_attributes = {}, {}
for c in components.models:
flag1, flag2 = components.get_script_options(c)
if flag2 is None:
parser.add_argument("--" + flag1,
action="store_true",
default=False,
help="Run ece2cmor3 exclusively for %s data" %
c)
else:
parser.add_argument("--" + flag1,
'-' + flag2,
action="store_true",
default=False,
help="Run ece2cmor3 exclusively for %s data" %
c)
model_attributes[c] = flag1
tabfile = components.models[c].get(components.table_file, "")
if tabfile:
option = os.path.basename(tabfile)
model_tabfile_attributes[c] = option
parser.add_argument("--" + option,
metavar="FILE.json",
type=str,
default=tabfile,
help="%s variable table (optional)" % c)
args = parser.parse_args()
modedict = {
"preserve": ece2cmorlib.PRESERVE,
"append": ece2cmorlib.APPEND,
"replace": ece2cmorlib.REPLACE
}
# Initialize ece2cmor:
ece2cmorlib.initialize(args.conf,
mode=modedict[args.mode],
tabledir=args.tabdir,
tableprefix=args.tabid)
ece2cmorlib.enable_masks = not args.nomask
ece2cmorlib.auto_filter = args.filter
# Fix exclusive run flags: if none are used, we cmorize for all components
model_active_flags = dict.fromkeys(components.models, False)
for model in model_attributes:
model_active_flags[model] = getattr(args, model_attributes[model],
False)
if not any(model_active_flags.values()):
model_active_flags = dict.fromkeys(model_active_flags, True)
# Load the variables as task targets:
for model in model_tabfile_attributes:
tabfile_attribute = model_tabfile_attributes[model]
attribute_value = getattr(args, tabfile_attribute, None)
if attribute_value is not None:
components.models[model][components.table_file] = attribute_value
taskloader.load_targets(args.vars, model_active_flags)
startdate = dateutil.parser.parse(args.date)
length = dateutil.relativedelta.relativedelta(months=1)
if model_active_flags["ifs"]:
refdate = dateutil.parser.parse(args.refd) if args.refd else None
# Execute the atmosphere cmorization:
ece2cmorlib.perform_ifs_tasks(args.datadir,
args.exp,
startdate,
length,
refdate=refdate,
outputfreq=args.freq,
tempdir=args.tmpdir,
taskthreads=args.npp,
cdothreads=args.ncdo,
maxsizegb=args.tmpsize)
if model_active_flags["nemo"]:
ece2cmorlib.perform_nemo_tasks(args.datadir, args.exp, startdate,
length)
# if procNEWCOMPONENT:
# ece2cmorlib.perform_NEWCOMPONENT_tasks(args.datadir, args.exp, startdate, length)
ece2cmorlib.finalize()
def init():
ece2cmorlib.initialize()
def main(args = None):
if args is None:
args = sys.argv[1:]
varlist_path_default = os.path.join(os.path.dirname(__file__),"resources","varlist.json")
parser = argparse.ArgumentParser(description = "Post-processing and cmorization of EC-Earth output",
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("datadir", metavar = "DIR", type = str)
parser.add_argument("date", metavar = "YYYY-mm-dd", type = str)
parser.add_argument("--vars", metavar = "FILE" , type = str, default = varlist_path_default,help = "File (json|f90 namelist|xlsx) containing cmor variables")
parser.add_argument("--conf", metavar = "FILE.json", type = str, default = ece2cmorlib.conf_path_default,help = "Input metadata file")
parser.add_argument("--exp", metavar = "EXPID", type = str, default = "ECE3",help = "Experiment prefix")
parser.add_argument("--refd", metavar = "YYYY-mm-dd", type = str, default = None,help = "Reference date (for atmosphere data), by default the start date")
parser.add_argument("--mode", metavar = "MODE", type = str, default = "preserve",help = "CMOR netcdf mode",choices = ["preserve","replace","append"])
parser.add_argument("--freq", metavar = "N", type = int, default = 3,help = "IFS output frequency, in hours (not required if autofilter is used)")
parser.add_argument("--tabdir", metavar = "DIR", type = str, default = ece2cmorlib.table_dir_default,help = "Cmorization table directory")
parser.add_argument("--tabid", metavar = "PREFIX", type = str, default = ece2cmorlib.prefix_default,help = "Cmorization table prefix string")
parser.add_argument("--tmpdir", metavar = "DIR", type = str, default = "/tmp/ece2cmor3", help = "Temporary working directory")
parser.add_argument("--npp", metavar = "N", type = int, default = 8, help = "Number of post-processing threads")
parser.add_argument("--tmpsize",metavar = "X", type = float, default = float("inf"),help = "Size of tempdir (in GB) that triggers flushing")
parser.add_argument("--ncdo", metavar = "N", type = int, default = 4,help = "Number of available threads per CDO postprocessing task")
parser.add_argument("-a", "--atm", action = "store_true", default = False, help = "Run ece2cmor3 exclusively for atmosphere data")
parser.add_argument("-o", "--oce", action = "store_true", default = False, help = "Run ece2cmor3 exclusively for ocean data")
parser.add_argument("--nomask" , action = "store_true", default = False, help = "Disable masking of fields")
parser.add_argument("--filter" , action = "store_true", default = False, help = "Automatic filtering of grib files")
args = parser.parse_args()
modedict = {"preserve":ece2cmorlib.PRESERVE,"append":ece2cmorlib.APPEND,"replace":ece2cmorlib.REPLACE}
# Initialize ece2cmor:
ece2cmorlib.initialize(args.conf,mode = modedict[args.mode],tabledir = args.tabdir,tableprefix = args.tabid)
ece2cmorlib.enable_masks = not args.nomask
ece2cmorlib.auto_filter = args.filter
# Fix conflicting flags
procatmos,prococean = not args.oce,not args.atm
if(not procatmos and not prococean):
procatmos,prococean = True,True
# Load the variables as task targets:
taskloader.load_targets(args.vars,load_atm_tasks = procatmos,load_oce_tasks = prococean)
startdate = dateutil.parser.parse(args.date)
length = dateutil.relativedelta.relativedelta(months = 1)
if(procatmos):
refdate = dateutil.parser.parse(args.refd) if args.refd else None
# Execute the atmosphere cmorization:
ece2cmorlib.perform_ifs_tasks(args.datadir,args.exp,startdate,length,refdate = refdate,
outputfreq = args.freq,
tempdir = args.tmpdir,
taskthreads = args.npp,
cdothreads = args.ncdo,
maxsizegb = args.tmpsize)
if(prococean):
ece2cmorlib.perform_nemo_tasks(args.datadir,args.exp,startdate,length)
ece2cmorlib.finalize()
def test_lookup_target():
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
assert tgt is not None
ece2cmorlib.finalize()
def test_initialize():
ece2cmorlib.initialize()
ece2cmorlib.finalize()
def test_init():
ece2cmorlib.initialize()
def main(args=None):
if args is None:
pass
formatter = lambda prog: argparse.ArgumentDefaultsHelpFormatter(
prog, max_help_position=30)
parser = argparse.ArgumentParser(
description="Post-processing and cmorization of EC-Earth output",
formatter_class=formatter)
required = parser.add_argument_group("required arguments")
parser.add_argument(
"datadir",
metavar="DIR",
type=str,
help="EC-Earth data directory, i.e. for a given component, "
"for a given leg")
parser.add_argument("--exp",
metavar="EXPID",
type=str,
default="ECE3",
help="Experiment prefix")
varsarg = required.add_mutually_exclusive_group(required=True)
varsarg.add_argument(
"--varlist",
metavar="FILE",
type=str,
help=
"File (json) containing cmor variables grouped per table, grouped per EC-Earth component"
)
varsarg.add_argument(
"--drq",
metavar="FILE",
type=str,
help=
"File (json|f90 namelist|xlsx) containing cmor variables, grouped per table"
)
required.add_argument("--meta",
metavar="FILE.json",
type=str,
required=True,
help="Input metadata file")
parser.add_argument("--odir",
metavar="DIR",
type=str,
default=None,
help="Output directory, by default the "
"metadata \'outpath\' entry")
cmor_utils.ScriptUtils.add_model_exclusive_options(parser, "ece2cmor")
parser.add_argument(
"--ececonf",
metavar='|'.join(components.ece_configs.keys()),
type=str,
help="EC-Earth configuration (only used with --drq option)")
parser.add_argument("--refd",
metavar="YYYY-mm-dd",
type=str,
default="1850-01-01",
help="Reference date for output time axes")
parser.add_argument("--npp",
metavar="N",
type=int,
default=8,
help="Number of parallel tasks (only relevant for "
"IFS cmorization")
parser.add_argument("--log",
action="store_true",
default=False,
help="Write to log file")
parser.add_argument("--flatdir",
action="store_true",
default=False,
help="Do not create sub-directories in "
"output folder")
parser.add_argument("--tabledir",
metavar="DIR",
type=str,
default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tableprefix",
metavar="PREFIX",
type=str,
default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
parser.add_argument("--tmpdir",
metavar="DIR",
type=str,
default="/tmp/ece2cmor",
help="Temporary working directory")
parser.add_argument(
"--overwritemode",
metavar="MODE",
type=str,
default="preserve",
help="MODE:preserve|replace|append, CMOR netcdf overwrite mode",
choices=["preserve", "replace", "append"])
parser.add_argument("--skip_alevel_vars",
action="store_true",
default=False,
help="Prevent loading atmospheric "
"model-level variables")
parser.add_argument(
"-V",
"--version",
action="version",
version="%(prog)s {version}".format(version=__version__.version))
# Deprecated arguments, only for backward compatibility
parser.add_argument("--ncdo",
metavar="N",
type=int,
default=4,
help=argparse.SUPPRESS)
parser.add_argument("--nomask",
action="store_true",
default=False,
help=argparse.SUPPRESS)
parser.add_argument("--nofilter",
action="store_true",
default=False,
help=argparse.SUPPRESS)
parser.add_argument("--atm",
action="store_true",
default=False,
help="Deprecated! Use --ifs instead")
parser.add_argument("--oce",
action="store_true",
default=False,
help="Deprecated! Use --nemo instead")
parser.add_argument("--conf",
action="store_true",
help="Deprecated! Use --meta instead")
parser.add_argument("--vars",
action="store_true",
help="Deprecated! Use --varlist instead")
cmor_utils.ScriptUtils.add_model_tabfile_options(parser)
args = parser.parse_args()
cmor_utils.ScriptUtils.set_custom_tabfiles(args)
logfile = None
logformat = "%(asctime)s %(levelname)s:%(name)s: %(message)s"
logdateformat = "%Y-%m-%d %H:%M:%S"
if getattr(args, "log", False):
dirs = os.path.abspath(args.datadir).split(os.sep)
fname = '-'.join([args.exp] + dirs[-2:] +
[time.strftime("%Y%m%d%H%M%S", time.gmtime())])
logfile = '.'.join([fname, "log"])
logging.basicConfig(filename=logfile,
level=logging.DEBUG,
format=logformat,
datefmt=logdateformat)
else:
logging.basicConfig(level=logging.DEBUG,
format=logformat,
datefmt=logdateformat)
if not os.path.isdir(args.datadir):
log.fatal("Your data directory argument %s cannot be found." %
args.datadir)
sys.exit(' Exiting ece2cmor.')
if args.varlist is not None and not os.path.isfile(args.varlist):
log.fatal("Your variable list json file %s cannot be found." %
args.varlist)
sys.exit(' Exiting ece2cmor.')
if args.drq is not None and not os.path.isfile(args.drq):
log.fatal("Your data request file %s cannot be found." % args.drq)
sys.exit(' Exiting ece2cmor.')
if not os.path.isfile(args.meta):
log.fatal("Your metadata file %s cannot be found." % args.meta)
sys.exit(' Exiting ece2cmor.')
modedict = {
"preserve": ece2cmorlib.PRESERVE,
"append": ece2cmorlib.APPEND,
"replace": ece2cmorlib.REPLACE
}
# Initialize ece2cmor:
ece2cmorlib.initialize(args.meta,
mode=modedict[args.overwritemode],
tabledir=args.tabledir,
tableprefix=args.tableprefix,
outputdir=args.odir,
logfile=logfile,
create_subdirs=(not args.flatdir))
ece2cmorlib.enable_masks = not args.nomask
ece2cmorlib.auto_filter = not args.nofilter
active_components = cmor_utils.ScriptUtils.get_active_components(
args, args.ececonf)
filters = None
if args.skip_alevel_vars:
def ifs_model_level_variable(target):
zaxis, levs = cmor_target.get_z_axis(target)
return zaxis not in ["alevel", "alevhalf"]
filters = {"model level": ifs_model_level_variable}
try:
if getattr(args, "varlist", None) is not None:
taskloader.load_tasks(args.varlist,
active_components=active_components,
target_filters=filters,
check_duplicates=True)
else:
taskloader.load_tasks_from_drq(args.drq,
active_components=["ifs"],
target_filters=filters,
check_prefs=True)
except taskloader.SwapDrqAndVarListException as e:
log.error(e.message)
opt1, opt2 = "vars" if e.reverse else "drq", "drq" if e.reverse else "vars"
log.error(
"It seems you are using the --%s option where you should use the --%s option for this file"
% (opt1, opt2))
sys.exit(' Exiting ece2cmor.')
refdate = datetime.datetime.combine(dateutil.parser.parse(args.refd),
datetime.datetime.min.time())
if "ifs" in active_components:
ece2cmorlib.perform_ifs_tasks(args.datadir,
args.exp,
refdate=refdate,
tempdir=args.tmpdir,
taskthreads=args.npp,
cdothreads=args.ncdo)
if "nemo" in active_components:
ece2cmorlib.perform_nemo_tasks(args.datadir, args.exp, refdate)
if "lpjg" in active_components:
ece2cmorlib.perform_lpjg_tasks(args.datadir, args.tmpdir, args.exp,
refdate)
if "tm5" in active_components:
ece2cmorlib.perform_tm5_tasks(args.datadir, args.tmpdir, args.exp,
refdate)
# if procNEWCOMPONENT in active_components:
# ece2cmorlib.perform_NEWCOMPONENT_tasks(args.datadir, args.exp, refdate)
ece2cmorlib.finalize()
