def test_duplicate_task(self):
ece2cmorlib_tests.init()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src1 = cmor_source.ifs_source.read("49.128")
tsk1 = cmor_task.cmor_task(src1, tgt)
ece2cmorlib.add_task(tsk1)
src2 = cmor_source.ifs_source.read("79.128")
tsk2 = cmor_task.cmor_task(src2, tgt)
ece2cmorlib.add_task(tsk2)
eq_(len(ece2cmorlib.tasks), 1)
ok_(tsk2 in ece2cmorlib.tasks)
ece2cmorlib.finalize()
def test_duplicate_task():
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src1 = cmor_source.ifs_source.read("49.128")
tsk1 = cmor_task.cmor_task(src1, tgt)
ece2cmorlib.add_task(tsk1)
src2 = cmor_source.ifs_source.read("79.128")
tsk2 = cmor_task.cmor_task(src2, tgt)
ece2cmorlib.add_task(tsk2)
assert len(ece2cmorlib.tasks) == 1
assert tsk2 in ece2cmorlib.tasks
ece2cmorlib.finalize()
def test_expr_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon")
src = cmor_source.ifs_source.read("214.128",
"sqrt(sqr(var165)+sqr(var166))")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk])
filepath = os.path.join(tmp_path, "166.128.105_165.128.105.3")
assert os.path.isfile(filepath)
assert getattr(tsk, cmor_task.filter_output_key) == [filepath]
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
assert param in [165, 166]
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
assert newdate == date + 1
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
assert newtime == (time + 300) % 2400
time = newtime
os.remove(filepath)
def test_surf_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "79.128.1")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, cmor_task.output_path_key), filepath)
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
eq_(param, 79)
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
eq_(newtime, (time + 300) % 2400)
time = newtime
os.remove(filepath)
def test_expr_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon")
src = cmor_source.ifs_source.read(
"var214=sqrt(sqr(var165)+sqr(var166))")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "165.128.105_166.128.105")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, "path"), filepath)
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
ok_(param in [165, 166])
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
eq_(newtime, (time + 300) % 2400)
time = newtime
os.remove(filepath)
def test_postproc_specmean():
testdata = os.path.dirname(
__file__) + "/test_data/ifsdata/6hr/ICMSHECE3+199001"
if test_utils.is_lfs_ref(testdata):
logging.info(
"Skipping test_postproc_specmean, download test data from lfs first"
)
return
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(130, 128)
target = [
t for t in targets if t.variable == "ta" and t.table == "Amon"
][0]
task = cmor_task.cmor_task(source, target)
postproc.mode = postproc.skip
ifs2cmor.temp_dir_ = os.getcwd()
ifs2cmor.ifs_spectral_file_ = testdata
ifs2cmor.postprocess([task])
path = os.path.join(os.getcwd(), "ta_Amon.nc")
nose.tools.eq_(getattr(task, "path"), path)
nose.tools.eq_(
getattr(task, "cdo_command"),
"-sp2gpl -monmean -sellevel,100000.,92500.,85000.,70000.,"
"60000.,50000.,40000.,30000.,25000.,20000.,15000.,10000.,7000.,"
"5000.,3000.,2000.,1000.,500.,100. "
"-selzaxis,pressure -selcode,130")
def get_sp_tasks(tasks, autofilter):
global ifs_spectral_file_
tasks_by_freq = cmor_utils.group(tasks, lambda task: task.target.frequency)
result = []
for freq, task_group in tasks_by_freq.iteritems():
tasks3d = [
t for t in task_group
if "alevel" in getattr(t.target, cmor_target.dims_key).split()
]
if not any(tasks3d):
continue
surf_pressure_tasks = [
t for t in task_group
if t.source.get_grib_code() == surface_pressure
and getattr(t, "time_operator", "point") in ["mean", "point"]
]
surf_pressure_task = surf_pressure_tasks[0] if any(
surf_pressure_tasks) else None
if surf_pressure_task:
result.append(surf_pressure_task)
else:
source = cmor_source.ifs_source(surface_pressure)
surf_pressure_task = cmor_task.cmor_task(
source, cmor_target.cmor_target("sp", freq))
setattr(surf_pressure_task.target, cmor_target.freq_key, freq)
setattr(surf_pressure_task.target, "time_operator", ["point"])
find_sp_variable(surf_pressure_task, autofilter)
result.append(surf_pressure_task)
for task3d in tasks3d:
setattr(task3d, "sp_task", surf_pressure_task)
return result
def test_surf_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk])
filepath = os.path.join(tmp_path, "79.128.1.3")
assert os.path.isfile(filepath)
assert getattr(tsk, cmor_task.filter_output_key) == [filepath]
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
assert param == 79
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
assert newdate == date + 1
date = newdate
newtime = reader.get_field(grib_file.time_key)
assert newtime == (time + 300) % 2400
time = newtime
os.remove(filepath)
def test_pressure_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("ua", "Amon")
src = cmor_source.ifs_source.read("131.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "131.128.210.6")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, cmor_task.filter_output_key), [filepath])
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
eq_(param, 131)
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
eq_(newtime, (time + 600) % 2400)
time = newtime
os.remove(filepath)
def perform_ifs_tasks(datadir, expname,
refdate=None,
postprocmode=postproc.recreate,
tempdir="/tmp/ece2cmor",
taskthreads=4,
cdothreads=4):
global log, tasks, table_dir, prefix, masks
validate_setup_settings()
validate_run_settings(datadir, expname)
ifs_tasks = [t for t in tasks if t.source.model_component() == "ifs"]
log.info("Selected %d IFS tasks from %d input tasks" % (len(ifs_tasks), len(tasks)))
if len(ifs_tasks) == 0:
return
tableroot = os.path.join(table_dir, prefix)
if enable_masks:
ifs2cmor.masks = {k: masks[k] for k in masks if masks[k]["source"].model_component() == "ifs"}
else:
ifs2cmor.masks = {}
if (not ifs2cmor.initialize(datadir, expname, tableroot, refdate if refdate else datetime.datetime(1850, 1, 1),
tempdir=tempdir, autofilter=auto_filter)):
return
postproc.postproc_mode = postprocmode
postproc.cdo_threads = cdothreads
area_task = cmor_task.cmor_task(cmor_source.ifs_source(cmor_source.grib_code(129)),
get_cmor_target("areacella", "fx"))
ifs2cmor.execute(ifs_tasks + [area_task], nthreads=taskthreads)
def get_sp_tasks(tasks):
tasks_by_freq = cmor_utils.group(
tasks, lambda task: (task.target.frequency, '_'.join(
getattr(task.target, "time_operator", ["mean"]))))
result = []
for freq, task_group in tasks_by_freq.iteritems():
tasks3d = [
t for t in task_group
if "alevel" in getattr(t.target, cmor_target.dims_key).split()
]
if not any(tasks3d):
continue
surf_pressure_tasks = [
t for t in task_group
if t.source.get_grib_code() == surface_pressure
]
if len(surf_pressure_tasks) > 0:
surf_pressure_task = surf_pressure_tasks[0]
result.append(surf_pressure_task)
else:
source = cmor_source.ifs_source(surface_pressure)
surf_pressure_task = cmor_task.cmor_task(
source, cmor_target.cmor_target("sp", tasks3d[0].target.table))
setattr(surf_pressure_task.target, cmor_target.freq_key, freq[0])
setattr(surf_pressure_task.target, "time_operator",
freq[1].split('_'))
setattr(surf_pressure_task.target, cmor_target.dims_key,
"latitude longitude")
find_sp_variable(surf_pressure_task)
result.append(surf_pressure_task)
for task3d in tasks3d:
setattr(task3d, "sp_task", surf_pressure_task)
return result
def get_mask_tasks(tasks):
global log, masks
selected_masks = []
for task in tasks:
msk = getattr(task.target, cmor_target.mask_key, None)
if msk:
if msk not in masks:
log.warning(
"Mask %s is not supported as an IFS mask, skipping masking"
% msk)
delattr(task.target, cmor_target.mask_key)
else:
selected_masks.append(msk)
continue
for area_operator in getattr(task.target, "area_operator", []):
words = area_operator.split()
if len(words) == 3 and words[1] == "where":
mask_name = words[2]
if mask_name not in masks:
log.warning(
"Mask %s is not supported as an IFS mask, skipping masking"
% mask_name)
else:
selected_masks.append(mask_name)
setattr(task.target, cmor_target.mask_key, mask_name)
result = []
for m in set(selected_masks):
target = cmor_target.cmor_target(m, "fx")
setattr(target, cmor_target.freq_key, 0)
setattr(target, "time_operator", ["point"])
result_task = cmor_task.cmor_task(masks[m]["source"], target)
setattr(result_task, cmor_task.output_path_key, ifs_gridpoint_file_)
result.append(result_task)
return result
def test_postproc_wap500():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(135, 128)
target = [t for t in targets if t.variable == "wap500" and t.table == "CFday"][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
nose.tools.eq_(command.create_command(), "-sp2gpl -daymean -sellevel,50000. -selzaxis,pressure -selcode,135")
def test_postproc_tasmax():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(201, 128)
target = [t for t in targets if t.variable == "tasmax" and t.table == "Amon"][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
nose.tools.eq_(command.create_command(), "-monmean -daymax -setgridtype,regular -selcode,201")
def test_postproc_gridmean():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(79, 128)
target = [t for t in targets if t.variable == "clwvi" and t.table == "CFday"][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
nose.tools.eq_(command.create_command(), "-setgridtype,regular -daymean -selcode,79")
def test_create_task(self):
ece2cmorlib_tests.init()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
ece2cmorlib.add_task(tsk)
ok_(tsk in ece2cmorlib.tasks)
ece2cmorlib.finalize()
def test_create_task():
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
ece2cmorlib.add_task(tsk)
assert tsk in ece2cmorlib.tasks
ece2cmorlib.finalize()
def create_cmor_task(pardict, target, tag):
global log, IFS_source_tag, Nemo_source_tag, json_source_key, json_grid_key
task = cmor_task.cmor_task(create_cmor_source(pardict, tag), target)
mask = pardict.get(json_masked_key, None)
if mask: setattr(task.target, cmor_target.mask_key, mask)
conv = pardict.get(cmor_task.conversion_key, None)
if conv: setattr(task, cmor_task.conversion_key, conv)
return task
def test_postproc_maxwindspeed():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.read("var88=sqrt(sqr(var165)+sqr(var166))")
target = [t for t in targets if t.variable == "sfcWindmax" and t.table == "day"][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
nose.tools.eq_(command.create_command(),
"-daymax -expr,'var88=sqrt(sqr(var165)+sqr(var166))' -setgridtype,regular -selcode,165,166")
def test_postproc_gridmeans():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source1 = cmor_source.ifs_source.create(79, 128)
target1 = [
t for t in targets if t.variable == "clwvi" and t.table == "CFday"
][0]
task1 = cmor_task.cmor_task(source1, target1)
source2 = cmor_source.ifs_source.create(164, 128)
target2 = [
t for t in targets if t.variable == "clt" and t.table == "day"
][0]
task2 = cmor_task.cmor_task(source2, target2)
postproc.mode = postproc.skip
ifs2cmor.temp_dir_ = os.getcwd()
ifs2cmor.postprocess([task1, task2])
path = os.path.join(os.getcwd(), "clt_day.nc")
nose.tools.eq_(getattr(task2, "path"), path)
nose.tools.eq_(getattr(task1, "cdo_command"),
"-setgridtype,regular -daymean -selcode,79")
def test_postproc_daymax():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(165, 128)
target = [
t for t in targets
if t.variable == "sfcWindmax" and t.table == "day"
][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
assert command.create_command(
) == "-daymax -setgridtype,regular -selcode,165"
def test_postproc_tasmax_missval():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(201, 128)
target = [
t for t in targets if t.variable == "tasmax" and t.table == "Amon"
][0]
task = cmor_task.cmor_task(source, target)
setattr(task, "missval", "0")
command = postproc.create_command(task)
assert command.create_command(
) == "-setmisstoc,0 -monmean -daymax -setgridtype,regular -selcode,201"
def test_postproc_specmean():
testdata = os.path.dirname(__file__) + "/test_data/ifsdata/6hr/ICMSHECE3+199001"
if test_utils.is_lfs_ref(testdata):
logging.info("Skipping test_postproc_specmean, download test data from lfs first")
return
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(131, 128)
target = [t for t in targets if t.variable == "ua" and t.table == "CFday"][0]
task = cmor_task.cmor_task(source, target)
setattr(task, "path", testdata)
command = postproc.create_command(task)
nose.tools.eq_(command.create_command(), "-sp2gpl -daymean -selzaxis,hybrid -selcode,131")
def test_validate_tasks():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt1 = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src1 = cmor_source.ifs_source.read("79.128")
tsk1 = cmor_task.cmor_task(src1, tgt1)
tgt2 = ece2cmorlib.get_cmor_target("ua", "Amon")
src2 = cmor_source.ifs_source.read("131.128")
tsk2 = cmor_task.cmor_task(src2, tgt2)
valid_tasks, varstasks = grib_filter.validate_tasks([tsk1, tsk2])
assert valid_tasks == [tsk1, tsk2]
key1 = (79, 128, grib_file.surface_level_code, 0,
cmor_source.ifs_grid.point)
key2 = (131, 128, grib_file.pressure_level_Pa_code, 92500,
cmor_source.ifs_grid.spec)
assert varstasks[key1] == [tsk1]
assert varstasks[key2] == [tsk2]
ltype, plevs = cmor_target.get_z_axis(tgt2)
levs = sorted([float(p) for p in plevs])
levcheck = sorted([k[3] for k in varstasks if k[0] == 131])
assert levs == levcheck
def test_postproc_post_expr():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.read("var23",
"var126020/var126022",
expr_order=1)
target = [
t for t in targets if t.variable == "cdnc" and t.table == "AERmon"
][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
assert command.create_command() == "-setgridtype,regular -expr,'var23=var20/var22' -monmean -selzaxis,hybrid " \
"-selcode,20,22"
def test_postproc_windspeed():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.read("var88",
"sqrt(sqr(var165)+sqr(var166))")
target = [
t for t in targets
if t.variable == "sfcWind" and t.table == "6hrPlevPt"
][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
assert command.create_command() == "-setgridtype,regular -expr,'var88=sqrt(sqr(var165)+sqr(var166))' " \
"-selhour,0,6,12,18 -selcode,165,166"
def test_postproc_mrsol():
source = cmor_source.ifs_source.read(
"118.129", "merge(70*var39,210*var40,720*var41,1890*var42)")
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
target = [
t for t in targets if t.variable == "mrsol" and t.table == "Eday"
][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
assert command.create_command() == "-setgridtype,regular -setcode,118 -daymean -expr," \
"'var1=70*var39;var2=210*var40;var3=720*var41;var4=1890*var42' -selcode," \
"39,40,41,42"
def create_cmor_task(pardict, target, tag):
global log, json_source_key, json_grid_key
source = create_cmor_source(pardict, tag)
if source is None:
log.error("Failed to construct a source for target variable %s in table %s...skipping task"
% (target.variable, target.table))
task = cmor_task.cmor_task(source, target)
mask = pardict.get(json_masked_key, None)
if mask:
setattr(task.target, cmor_target.mask_key, mask)
conv = pardict.get(cmor_task.conversion_key, None)
if conv:
setattr(task, cmor_task.conversion_key, conv)
return task
def create_cmor_task(pardict, target, component):
global log, json_source_key
source = create_cmor_source(pardict, component)
if source is None:
raise ValueError("Failed to construct a source for target variable %s in table %s: task skipped."
% (target.variable, target.table))
task = cmor_task.cmor_task(source, target)
mask = pardict.get(json_masked_key, None)
if mask:
setattr(task.target, cmor_target.mask_key, mask)
for par in pardict:
if par not in [json_source_key, json_target_key, json_mask_key, json_masked_key, json_table_key, "expr"]:
setattr(task, par, pardict[par])
return task
def test_postproc_mrsol_masked():
source = cmor_source.ifs_source.read(
"118.129",
"merge(70*var39,210*var40,720*var41,1890*var42)",
mask_expr="var172>=0.5")
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
target = [
t for t in targets if t.variable == "mrsol" and t.table == "Eday"
][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
assert command.create_command() == "-setgridtype,regular -setcode,118 -daymean -expr,'var1=(70*var39)/(" \
"var172>=0.5);var2=(210*var40)/(var172>=0.5);var3=(720*var41)/(" \
"var172>=0.5);var4=(1890*var42)/(var172>=0.5)' -selcode,39,40,41,42,172"